kuroko2 0.2.0

data/app/assets/stylesheets/kuroko2/fonts.scss

@@ -0,0 +1 @@
+@import url('//fonts.googleapis.com/css?family=Inconsolata|Source+Sans+Pro:300,400,600,300italic,400italic,600italic|Homenaje');

data/app/assets/stylesheets/kuroko2/job_definitions.scss

@@ -0,0 +1,19 @@
+.description {
+  h1 {
+    font-size: 16px;
+  }
+  h2, h3, h4 {
+    font-weight: bold;
+    font-size: 14px;
+  }
+}
+
+.star-holder {
+  a:link, a:visited, a:hover, a:active {
+    color: #444;
+  }
+}
+
+.script-input {
+  font-family: 'Inconsolata';
+}

data/app/assets/stylesheets/kuroko2/users.scss

@@ -0,0 +1,7 @@
+.user-controls {
+  text-align: right;
+
+  a:not(:last-child) {
+    margin-right: 5px;
+  }
+}

data/app/controllers/kuroko2/api/application_controller.rb

@@ -0,0 +1,49 @@
+class Kuroko2::Api::ApplicationController < ActionController::Base
+  include Garage::ControllerHelper
+
+  before_action :api_authentication
+
+  rescue_from ActiveRecord::RecordNotFound do |exception|
+    respond_with_error(404, 'record_not_found', exception.message)
+  end
+
+  rescue_from HTTP::Forbidden do |exception|
+    respond_with_error(403, 'forbidden', exception.message)
+  end
+
+  rescue_from HTTP::Unauthorized do |exception|
+    respond_with_error(401, 'unauthorized', exception.message)
+  end
+
+  rescue_from WeakParameters::ValidationError do |exception|
+    respond_with_error(400, 'bad_request', exception.message)
+  end
+
+  private
+
+  # @param [Integer] status_code HTTP status code
+  # @param [String] error_code Must be unique
+  # @param [String] message Error message for API client, not for end user.
+  def respond_with_error(status_code, error_code, message)
+    render json: { status_code: status_code, error_code: error_code, message: message }, status: status_code
+  end
+
+  def api_authentication
+    service_name = authenticate_with_http_basic do |name, api_key|
+      stored = Kuroko2.config.api_basic_authentication_applications.try!(name.to_sym)
+      if Rack::Utils.secure_compare(api_key, stored)
+        name.to_sym
+      else
+        nil
+      end
+    end
+
+    if service_name.nil?
+      raise HTTP::Unauthorized
+    end
+  end
+
+  def basic_user_name
+    ActionController::HttpAuthentication::Basic.user_name_and_password(request).first
+  end
+end

data/app/controllers/kuroko2/api/job_instances_controller.rb

@@ -0,0 +1,46 @@
+class Kuroko2::Api::JobInstancesController < Kuroko2::Api::ApplicationController
+  include Garage::RestfulActions
+
+  validates :create do
+    hash :env, description: 'Env variables to launch an instance' do |env|
+      env.to_h.all? { |_, v| v.is_a?(String) }
+    end
+  end
+
+  private
+
+  def require_resources
+    protect_resource_as Kuroko2::Api::JobInstanceResource
+  end
+
+  def create_resource
+    definition = Kuroko2::JobDefinition.find(params[:job_definition_id])
+    unless definition.api_allowed?
+      raise HTTP::Forbidden.new("#{definition.name} is not allowed to be executed via API")
+    end
+
+    instance = definition.job_instances.create(
+      script: definition.script.prepend(env_script),
+    )
+    instance.logs.info("Launched by instances API (#{basic_user_name})")
+    Kuroko2::Api::JobInstanceResource.new(instance)
+  end
+
+  def require_resource
+    instance = Kuroko2::JobInstance.find(params[:id])
+    @resource = Kuroko2::Api::JobInstanceResource.new(instance)
+  end
+
+  def env_script
+    return '' unless params[:env]
+
+    params[:env].permit!.to_h.map { |key, value|
+      "env: #{key}='#{value.gsub("'", "\\\\'")}'"
+    }.join("\n").concat("\n")
+  end
+
+  # Don't set location header
+  def location
+    nil
+  end
+end

data/app/controllers/kuroko2/api/stats_controller.rb

@@ -0,0 +1,28 @@
+class Kuroko2::Api::StatsController < Kuroko2::Api::ApplicationController
+  skip_before_action :api_authentication
+
+  COUNT_INSTANCES_SQL = <<-SQL
+    SELECT COUNT(error_at IS NULL OR NULL)     AS `kuroko2.job_instances.working`,
+           COUNT(error_at IS NOT NULL OR NULL) AS `kuroko2.job_instances.error`
+    FROM #{Kuroko2::JobInstance.table_name}
+    WHERE finished_at IS NULL
+      AND canceled_at IS NULL
+  SQL
+
+  def instance
+    render json: ActiveRecord::Base.connection.select_one(COUNT_INSTANCES_SQL).to_hash
+  end
+
+  def waiting_execution
+    waiting_executions_count = Kuroko2::Execution.select('queue, COUNT(1) AS `count`').
+      where('started_at IS NULL AND created_at < ?', 3.minutes.ago).group(:queue)
+
+    render json: Kuroko2::Worker.pluck(:queue).uniq.inject({}) { |result, queue|
+      result.merge(
+        "kuroko2.executions.waiting.#{queue}" => waiting_executions_count.find { |count|
+          count.queue == queue
+        }.try!(:[], "count") || 0
+      )
+    }
+  end
+end

data/app/controllers/kuroko2/application_controller.rb

@@ -0,0 +1,43 @@
+class Kuroko2::ApplicationController < ActionController::Base
+  # Prevent CSRF attacks by raising an exception.
+  # For APIs, you may want to use :null_session instead.
+  protect_from_forgery with: :exception
+
+  helper_method :current_user, :signed_in?
+  before_action :require_sign_in
+
+  rescue_from HTTP::BadRequest do
+    respond_to do |format|
+      format.html { render 'public/500.html', layout: false, status: :bad_request }
+      format.json { render json: { message: 'Bad Request' }, status: :bad_request }
+    end
+  end
+
+  def current_user
+    @_current_user ||= begin
+      if (id = session[:user_id])
+        Kuroko2::User.active.find(id)
+      end
+    end
+  rescue ActiveRecord::RecordNotFound
+    reset_session
+    redirect_to sign_in_path(return_to: url_for(params.permit!.to_h.merge(only_path: true)))
+  end
+
+  private
+
+  def current_user=(user)
+    session[:user_id] = user.id
+    @_current_user    = user
+  end
+
+  def signed_in?
+    current_user.present?
+  end
+
+  def require_sign_in
+    unless signed_in?
+      redirect_to sign_in_path(return_to: url_for(params.permit!.to_h.merge(only_path: true)))
+    end
+  end
+end

data/app/controllers/kuroko2/dashboard_controller.rb

@@ -0,0 +1,27 @@
+class Kuroko2::DashboardController < Kuroko2::ApplicationController
+  def index
+    @definitions = current_user.job_definitions
+
+    @input_tags  = params[:tag] || []
+    if @input_tags.present?
+      @definitions = @definitions.tagged_by(@input_tags)
+    end
+
+    @instances    = Kuroko2::JobInstance.working.where(job_definition: @definitions)
+    @related_tags = @definitions.includes(:tags).map(&:tags).flatten.uniq
+
+  end
+
+  def osd
+    render xml: <<-XML.strip_heredoc
+    <?xml version="1.0" encoding="UTF-8" ?>
+    <OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
+    <ShortName>Kuroko2</ShortName>
+    <Description>Search Kuroko2</Description>
+    <InputEncoding>UTF-8</InputEncoding>
+    <Url type="text/html" method="get" template="#{job_definitions_url}?q={searchTerms}"/>
+    <Image width="16" height="16" type="image/x-icon">#{root_url}favicon.ico</Image>
+    </OpenSearchDescription>
+    XML
+  end
+end

data/app/controllers/kuroko2/execution_logs_controller.rb

@@ -0,0 +1,19 @@
+class Kuroko2::ExecutionLogsController < Kuroko2::ApplicationController
+  def index
+    @definition = Kuroko2::JobDefinition.find(logs_params[:job_definition_id])
+    @instance   = Kuroko2::JobInstance.find(logs_params[:job_instance_id])
+
+    execution_logger = Kuroko2::ExecutionLogger.
+      get_logger(stream_name: "JOB#{sprintf("%010d", @definition.id)}/#{@instance.id}")
+
+    @response = execution_logger.get_logs(logs_params[:token])
+  rescue Kuroko2::ExecutionLogger::NotFound
+    head 404
+  end
+
+  private
+
+  def logs_params
+    params.permit(:job_definition_id, :job_instance_id, :token)
+  end
+end

data/app/controllers/kuroko2/executions_controller.rb

@@ -0,0 +1,28 @@
+class Kuroko2::ExecutionsController < Kuroko2::ApplicationController
+
+  before_action :set_execution, only: %i(destroy)
+
+  def index
+    @executions = Kuroko2::Execution.where(finished_at: nil).order(created_at: :desc).includes(job_instance: :job_definition)
+  end
+
+  def destroy
+    if @execution.try(:pid)
+      hostname = Kuroko2::Worker.executing(@execution.id).try(:hostname)
+      Kuroko2::ProcessSignal.create!(pid: @execution.pid, hostname: hostname) if hostname
+    end
+
+    redirect_to job_definition_job_instance_path(job_definition_id: execution_params[:job_definition_id], id: execution_params[:job_instance_id])
+  end
+
+  private
+
+  def set_execution
+    @execution = Kuroko2::Execution.where(job_definition_id: execution_params[:job_definition_id], job_instance_id: execution_params[:job_instance_id], id: execution_params[:id]).take
+  end
+
+  def execution_params
+    params.permit(:id, :job_definition_id, :job_instance_id)
+  end
+
+end

data/app/controllers/kuroko2/job_definition_stats_controller.rb

@@ -0,0 +1,54 @@
+class Kuroko2::JobDefinitionStatsController < Kuroko2::ApplicationController
+  def index
+    set_definition
+  end
+
+  def execution_time
+    set_definition
+    set_period
+
+    @logs = Kuroko2::JobInstance.where(job_definition_id: params[:job_definition_id]).
+      order(created_at: :desc)
+    if @start_at
+      @logs = @logs.where(created_at: @start_at..@end_at)
+    else
+      @logs = @logs.limit(10)
+    end
+  end
+
+  def memory
+    set_definition
+    set_period
+
+    target_instance = Kuroko2::JobInstance.where(job_definition_id: params[:job_definition_id])
+    if @start_at
+      target_instance = target_instance.where(created_at: @start_at..@end_at)
+    end
+
+    @logs = Kuroko2::MemoryConsumptionLog.joins(:job_instance).
+      merge(target_instance).order(created_at: :desc)
+    @logs = @logs.limit(10) unless @start_at
+  end
+
+  private
+
+  def set_period
+    @end_at   = Time.now
+
+    @start_at =
+      case params[:period]
+      when /\A(\d+)d\z/
+        $1.to_i.days.ago(@end_at)
+      when /\A(\d+)w\z/
+        $1.to_i.weeks.ago(@end_at)
+      when /\A(\d+)m\z/
+        $1.to_i.month.ago(@end_at)
+      else
+        nil
+      end
+  end
+
+  def set_definition
+    @definition =  Kuroko2::JobDefinition.find(params[:job_definition_id])
+  end
+end

data/app/controllers/kuroko2/job_definitions_controller.rb

@@ -0,0 +1,100 @@
+class Kuroko2::JobDefinitionsController < Kuroko2::ApplicationController
+  before_action :set_definition, only: [:show, :edit, :update, :destroy]
+
+  def index
+    rel = Kuroko2::JobDefinition
+    query = query_params[:q]
+
+    if query.present?
+      rel = rel.search_by(query)
+    end
+
+    @input_tags = query_params[:tag] || []
+    if @input_tags.present?
+      rel = rel.tagged_by(@input_tags)
+    end
+
+    if query.present? || @input_tags.present?
+      @related_tags = rel.includes(:tags).map(&:tags).flatten.uniq
+    else
+      @related_tags = Kuroko2::Tag.all
+    end
+
+    @definitions = rel.ordered.page(page_params[:page])
+  end
+
+  def show
+    @instances = @definition.job_instances.page(0)
+    @schedules = @definition.job_schedules
+    @suspend_schedules = @definition.job_suspend_schedules
+
+    @schedule  = Kuroko2::JobSchedule.new(job_definition: @definitions)
+    @suspend_schedule = Kuroko2::JobSuspendSchedule.new(job_definition: @definitions)
+  end
+
+  def new
+    @definition = Kuroko2::JobDefinition.new
+  end
+
+  def create
+    @definition = Kuroko2::JobDefinition.new(definition_params)
+    @definition.admins << current_user
+
+    if @definition.save
+      current_user.stars.create(job_definition: @definition)
+
+      redirect_to @definition, notice: 'Job Definition was successfully created.'
+    else
+      render :new
+    end
+  end
+
+  def edit
+  end
+
+  def update
+    success = ActiveRecord::Base.transaction do
+      @definition.attributes = definition_params
+      @definition.admins     = Kuroko2::User.active.with(admin_id_params)
+
+      @definition.save
+    end
+
+    if success
+      redirect_to @definition, notice: 'Job Definition was successfully updated.'
+    else
+      render :edit
+    end
+  end
+
+  def destroy
+    @definition.destroy
+
+    redirect_to job_definitions_path, notice: 'Job Definition was successfully destroyed.'
+  end
+
+  private
+
+  def admin_id_params
+    params.require(:admin_assignments).permit(user_id: []).
+      try(:[], :user_id).
+      try(:reject!, &:blank?).
+      try(:map, &:to_i) || []
+  end
+
+  def definition_params
+    params.require(:job_definition).permit(:name, :description, :script, :notify_cancellation, :hipchat_room, :hipchat_notify_finished, :suspended, :prevent_multi, :prevent_multi_on_failure, :hipchat_additional_text, :text_tags, :api_allowed, :slack_channel)
+  end
+
+  def query_params
+    params.permit(:q, tag: [])
+  end
+
+  def page_params
+    params.permit(:page)
+  end
+
+  def set_definition
+    @definition = Kuroko2::JobDefinition.find(params[:id])
+  end
+end

data/app/controllers/kuroko2/job_instances_controller.rb

@@ -0,0 +1,87 @@
+class Kuroko2::JobInstancesController < Kuroko2::ApplicationController
+  before_action :set_definition, only: %i(index create show destroy force_destroy)
+  before_action :set_instance, only: %i(show destroy force_destroy)
+
+  def index
+    @instances = @definition.job_instances.page(page_params[:page])
+
+    render layout: false
+  end
+
+  def create
+    if params[:job_definition].present?
+      @instance = @definition.job_instances.create(params.require(:job_definition).permit(:script))
+    else
+      @instance = @definition.job_instances.create
+    end
+
+    if @instance
+      @instance.logs.info("Launched by #{current_user.name}.")
+
+      redirect_to job_definition_job_instance_path(@definition, @instance)
+    else
+      raise HTTP::BadRequest
+    end
+  end
+
+  def show
+    @logs   = @instance.logs
+    @tokens = @instance.tokens
+
+    if params[:mode] == :naked
+      render partial: 'instance', layout: false
+    else
+      # render
+    end
+  end
+
+  def destroy
+    if @instance.cancelable?
+      ActiveRecord::Base.transaction { @instance.cancel }
+
+      message = "This job was canceled by #{current_user.name}."
+      @instance.logs.warn(message)
+      Kuroko2.logger.warn(message)
+    end
+
+    redirect_to job_definition_job_instance_path(@definition, @instance)
+  end
+
+  def force_destroy
+    ActiveRecord::Base.transaction do
+      @instance.executions.each do |execution|
+        execution = Kuroko2::Worker.executing(execution.id)
+        execution.update_column(:execution_id, nil) if execution
+      end
+
+      @instance.cancel
+    end
+
+    message = "Force canceled by #{current_user.name}."
+    @instance.logs.warn(message)
+    Kuroko2.logger.warn(message)
+
+    redirect_to job_definition_job_instance_path(@definition, @instance)
+  end
+
+  def working
+    @instances = Kuroko2::JobInstance.working.order(id: :desc).joins(job_definition: :admins).includes(job_definition: :admins)
+  end
+
+  private
+  def job_instance_params
+    params.permit(:job_definition_id, :id)
+  end
+
+  def page_params
+    params.permit(:page)
+  end
+
+  def set_definition
+    @definition = Kuroko2::JobDefinition.find(job_instance_params[:job_definition_id])
+  end
+
+  def set_instance
+    @instance = @definition.job_instances.find(job_instance_params[:id])
+  end
+end

data/app/controllers/kuroko2/job_schedules_controller.rb

@@ -0,0 +1,39 @@
+class Kuroko2::JobSchedulesController < Kuroko2::ApplicationController
+  before_action :set_definition, only: %i(index create destroy)
+
+  def index
+    @schedules = @definition.job_schedules
+    @schedule  = Kuroko2::JobSchedule.new
+
+    render layout: false
+  end
+
+  def create
+    schedule = @definition.job_schedules.create(job_schedule_params)
+
+    if schedule.valid?
+      render json: schedule, status: :created
+    else
+      render json: schedule, status: :bad_request
+    end
+  end
+
+  def destroy
+    schedule = Kuroko2::JobSchedule.find(params[:id])
+    if schedule.destroy
+      render json: schedule, status: :ok
+    else
+      render json: schedule, status: :bad_request
+    end
+  end
+
+  private
+
+  def job_schedule_params
+    params.require(:job_schedule).permit(:cron)
+  end
+
+  def set_definition
+    @definition = Kuroko2::JobDefinition.find(params[:job_definition_id])
+  end
+end

data/app/controllers/kuroko2/job_suspend_schedules_controller.rb

@@ -0,0 +1,38 @@
+class Kuroko2::JobSuspendSchedulesController < Kuroko2::ApplicationController
+  before_action :set_definition, only: %i(index create destroy)
+
+  def index
+    @suspend_schedules = @definition.job_suspend_schedules
+    @suspend_schedule  = Kuroko2::JobSuspendSchedule.new
+    render layout: false
+  end
+
+  def create
+    suspend_schedule = @definition.job_suspend_schedules.create(job_suspend_schedule_params)
+
+    if suspend_schedule.valid?
+      render json: suspend_schedule, status: :created
+    else
+      render json: suspend_schedule, status: :bad_request
+    end
+  end
+
+  def destroy
+    suspend_schedule = Kuroko2::JobSuspendSchedule.find(params[:id])
+    if suspend_schedule.destroy
+      render json: suspend_schedule, status: :ok
+    else
+      render json: suspend_schedule, status: :bad_request
+    end
+  end
+
+  private
+
+  def job_suspend_schedule_params
+    params.require(:job_suspend_schedule).permit(:cron)
+  end
+
+  def set_definition
+    @definition = Kuroko2::JobDefinition.find(params[:job_definition_id])
+  end
+end

data/app/controllers/kuroko2/job_timelines_controller.rb

@@ -0,0 +1,56 @@
+class Kuroko2::JobTimelinesController < Kuroko2::ApplicationController
+  def index
+    find_user
+  end
+
+  def dataset
+    find_user
+    set_period
+
+    definition_ids = []
+    definition_ids << my_difinitions.pluck(:id)
+    definition_ids << assigned_definitions.pluck(:id)
+
+    @instances = Kuroko2::JobInstance.includes(:job_definition).
+      where(job_definition_id: definition_ids.flatten.uniq, created_at: @start_at..@end_at).
+      order(:created_at)
+  end
+
+  private
+
+  def my_difinitions
+    rel = @user.job_definitions
+    rel = rel.tagged_by(params[:tag]) if params[:tag].present?
+    rel
+  end
+
+  def assigned_definitions
+    rel = @user.assigned_job_definitions
+    rel = rel.tagged_by(params[:tag]) if params[:tag].present?
+    rel
+  end
+
+  def find_user
+    if params[:user_id]
+      @user = Kuroko2::User.find(params[:user_id])
+    else
+      @user = current_user
+    end
+  end
+
+  def set_period
+    @end_at   = Time.now
+
+    @start_at =
+      case params[:period]
+      when /\A(\d+)h\z/
+        $1.to_i.hours.ago(@end_at)
+      when /\A(\d+)d\z/
+        $1.to_i.days.ago(@end_at)
+      when /\A(\d+)w\z/
+        $1.to_i.weeks.ago(@end_at)
+      else
+        1.hour.ago(@end_at)
+      end
+  end
+end

data/app/controllers/kuroko2/logs_controller.rb

@@ -0,0 +1,15 @@
+class Kuroko2::LogsController < Kuroko2::ApplicationController
+  def index
+    @definition = Kuroko2::JobDefinition.find(logs_params[:job_definition_id])
+    @instance   = Kuroko2::JobInstance.find(logs_params[:job_instance_id])
+    @logs       = @instance.logs
+
+    render layout: false
+  end
+
+  private
+
+  def logs_params
+    params.permit(:job_definition_id, :job_instance_id)
+  end
+end