kuroko2 0.2.0

data/app/models/kuroko2/job_definition.rb

@@ -0,0 +1,147 @@
+class Kuroko2::JobDefinition < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  module PreventMultiStatus
+    NONE = 0
+    WORKING_OR_ERROR = 1
+    WORKING = 2
+    ERROR = 3
+  end
+
+  PREVENT_TOKEN_STATUSES = {
+    PreventMultiStatus::NONE => [],
+    PreventMultiStatus::WORKING_OR_ERROR => [
+      Kuroko2::Token::WORKING,
+      Kuroko2::Token::FAILURE,
+      Kuroko2::Token::CRITICAL
+    ],
+    PreventMultiStatus::WORKING => [Kuroko2::Token::WORKING],
+    PreventMultiStatus::ERROR => [Kuroko2::Token::FAILURE, Kuroko2::Token::CRITICAL],
+  }
+
+  self.locking_column = :version
+
+  paginates_per 100
+
+  has_many :admin_assignments, dependent: :destroy
+  has_many :admins, -> { active }, through: :admin_assignments, source: :user
+  has_many :job_instances, -> { order(:id).reverse_order } do
+    def any_token?
+      self.any? do |instance|
+        instance.tokens.present?
+      end
+    end
+  end
+  has_many :job_schedules, dependent: :delete_all
+  has_many :job_suspend_schedules, dependent: :delete_all
+  has_many :job_definition_tags
+  has_many :tags, through: :job_definition_tags
+  has_one :memory_expectancy, dependent: :destroy
+
+  before_destroy :confirm_active_instances
+  after_initialize :set_default_values
+  after_save :create_default_memory_expectancy, on: :create
+
+  scope :ordered, -> { order(:id) }
+  scope :tagged_by, ->(tags) {
+    where(
+      id: Kuroko2::JobDefinitionTag.
+        where(tag_id: Kuroko2::Tag.where(name: tags).pluck(:id)).
+        group(:job_definition_id).
+        having('COUNT(1) >= ?', tags.size).
+        pluck(:job_definition_id)
+    )
+  }
+  scope :search_by, ->(query) {
+    column = arel_table
+    or_query = column[:name].matches("%#{query}%").or(column[:script].matches("%#{query}%"))
+
+    search_by_tag_definition_ids = Kuroko2::JobDefinitionTag.joins(:tag).
+      where('tags.name LIKE ?', "%#{query}%").distinct.pluck(:job_definition_id)
+
+    if search_by_tag_definition_ids.present?
+      or_query = or_query.or(column[:id].in(search_by_tag_definition_ids))
+    end
+
+    where(or_query)
+  }
+
+
+  validates :name, length: { maximum: 40 }, presence: true
+  validates :description, presence: true
+  validates :script, presence: true
+  validate :script_syntax
+  validate :validate_number_of_admins
+  validates :hipchat_additional_text, length: { maximum: 180 }
+  validates :slack_channel, length: { maximum: 21 }, format: {
+    with: /\A#[^\.\s]+\z/, allow_blank: true,
+    message: ' must start with # and must not include any dots or spaces'
+  }
+
+  def proceed_multi_instance?
+    tokens = Kuroko2::Token.where(job_definition_id: self.id)
+    (tokens.map(&:status) & PREVENT_TOKEN_STATUSES[self.prevent_multi]).empty?
+  end
+
+  def text_tags
+    tags.pluck(:name).join(',')
+  end
+
+  def text_tags=(text_tags)
+    self.tags = text_tags.gsub(/[[:blank:]]+/, '').split(/[,、]/).uniq.map do |name|
+      Kuroko2::Tag.find_or_create_by(name: name)
+    end
+  end
+
+  private
+
+  def confirm_active_instances
+    if job_instances.any_token?
+      errors.add(:base, I18n.t('model.job_definition.confirm_active_instances'))
+      throw :abort
+    end
+  end
+
+  def set_default_values
+    self.description ||= <<-EOF.strip_heredoc
+      An description of the job definition.
+
+      ## Failure Affects
+      Affected users, services and/ or business areas.
+
+      ## Workaround
+      Choose one of the following:
+      - __Retry__ as soon as possible.
+      - Make an urgent call to administrator (Job stays in _Error_ state)
+      - Do nothing, and let administrator recover later (Job stays in _Error_ state)
+      - Ignore error and _Cancel_ the job (No recovery required)
+
+      ## Recovery Procedures
+      Describe how to recover from the failure.
+    EOF
+  end
+
+  def create_default_memory_expectancy
+    create_memory_expectancy! unless memory_expectancy
+  end
+
+  def script_syntax
+    Kuroko2::Workflow::ScriptParser.new(script).parse
+
+    true
+  rescue Kuroko2::Workflow::SyntaxError => e
+    errors.add(:base, I18n.t('model.job_definition.script_syntax', reason: e.message))
+
+    false
+  rescue Kuroko2::Workflow::AssertionError => e
+    errors.add(:base, I18n.t('model.job_definition.validation_error', reason: e.message))
+
+    false
+  end
+
+  def validate_number_of_admins
+    if self.admins.empty?
+      errors.add(:admins, I18n.t('model.job_definition.validate_number_of_admins'))
+    end
+  end
+end

data/app/models/kuroko2/job_definition_tag.rb

@@ -0,0 +1,6 @@
+class Kuroko2::JobDefinitionTag < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  belongs_to :tag
+  belongs_to :job_definition
+end

data/app/models/kuroko2/job_instance.rb

@@ -0,0 +1,118 @@
+class Kuroko2::JobInstance < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  belongs_to :job_definition
+
+  has_many :logs, dependent: :delete_all do
+    def info(message)
+      add('INFO', message)
+    end
+
+    def warn(message)
+      add('WARN', message)
+    end
+
+    def error(message)
+      add('ERROR', message)
+    end
+
+    private
+    def add(level, message)
+      self.create(level: level, message: message)
+    end
+  end
+  has_many :tokens, dependent: :restrict_with_exception
+  has_many :executions, dependent: :restrict_with_exception
+  has_one :memory_consumption_log, dependent: :destroy
+
+  before_create :copy_script
+  after_create :generate_token
+
+  scope :working, -> { where(finished_at: nil, canceled_at: nil) }
+  scope :finished, -> { where.not(finished_at: nil) }
+
+  def error?
+    working? && error_at?
+  end
+
+  def working?
+    !finished_at? && !canceled_at?
+  end
+
+  def cancelable?
+    tokens.first.try(:cancelable?)
+  end
+
+  def cancel
+    self.tokens.destroy(*self.tokens)
+    self.executions.destroy(*self.executions)
+    self.touch(:canceled_at)
+  end
+
+  # Log given value if it is greater than stored one.
+  # This logging is not so important that we can ignore race condition,
+  # so we use `#update` and `#create_association` without bang here.
+  # @param [Intger] value
+  def log_memory_consumption(value)
+    if memory_consumption_log
+      max = [value, memory_consumption_log.value].max
+      memory_consumption_log.update(value: max)
+    else
+      create_memory_consumption_log(value: value)
+    end
+  end
+
+  def execution_minutes
+    (((error_at || canceled_at || finished_at || Time.now) - created_at).to_f / 60).round(2)
+  end
+
+  def status
+    if finished_at?
+      'success'
+    elsif canceled_at?
+      'canceled'
+    elsif error_at?
+      'error'
+    else
+      'working'
+    end
+  end
+
+  private
+
+  def copy_script
+    self.script = job_definition.try(:script) if self.script.blank?
+  end
+
+  def generate_token
+    unless self.job_definition
+      raise 'No parent association is found'
+    end
+
+    if job_definition.proceed_multi_instance?
+      self.tokens << Kuroko2::Token.new do |token|
+        definition = self.job_definition
+
+        token.job_definition         = definition
+        token.job_definition_version = definition.version
+        token.script                 = self.script
+        token.context                = {
+          meta: {
+            launched_time:       Time.now,
+            job_definition_id:   definition.id,
+            job_definition_name: definition.name,
+            job_instance_id:     id,
+          }
+        }
+      end
+    else
+      self.touch(:canceled_at)
+
+      message = 'This job was canceled because there is already a working or erred job instance.'
+      self.logs.warn(message)
+      Kuroko2.logger.warn(message)
+
+      Kuroko2::Workflow::Notifier.notify(:cancellation, self)
+    end
+  end
+end

data/app/models/kuroko2/job_schedule.rb

@@ -0,0 +1,93 @@
+class Kuroko2::JobSchedule < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  belongs_to :job_definition
+
+  CRON_FORMAT = /\A
+    (?:[1-5]?[0-9]|(?:[1-5]?[0-9]\-[1-5]?[0-9]|\*)(?:\/[1-5]?[0-9])?)(?:,(?:[1-5]?[0-9]|(?:[1-5]?[0-9]\-[1-5]?[0-9]|\*)(?:\/[1-5]?[0-9])?))*
+    \s+
+    (?:1?[0-9]|2[0-3]|(?:(?:1?[0-9]|2[0-3])\-(?:1?[0-9]|2[0-3])|\*)(?:\/(?:1?[0-9]|2[0-3]))?)(?:,(?:1?[0-9]|2[0-3]|(?:(?:1?[0-9]|2[0-3])\-(?:1?[0-9]|2[0-3])|\*)(?:\/(?:1?[0-9]|2[0-3]))?))*
+    \s+
+    (?:(?:[1-9]|[1-2][0-9]|3[0-1])|(?:(?:[1-9]|[1-2][0-9]|3[0-1])\-(?:[1-9]|[1-2][0-9]|3[0-1])|\*)(?:\/(?:[1-9]|[1-2][0-9]|3[0-1]))?)(?:,(?:(?:[1-9]|[1-2][0-9]|3[0-1])|(?:(?:[1-9]|[1-2][0-9]|3[0-1])\-(?:[1-9]|[1-2][0-9]|3[0-1])|\*)(?:\/(?:[1-9]|[1-2][0-9]|3[0-1]))?))*
+    \s+
+    (?:(?:[1-9]|1[0-2])|(?:(?:[1-9]|1[0-2])\-(?:[1-9]|1[0-2])|\*)(?:\/(?:[1-9]|1[0-2]))?)(?:,(?:(?:[1-9]|1[0-2])|(?:(?:[1-9]|1[0-2])\-(?:[1-9]|1[0-2])|\*)(?:\/(?:[1-9]|1[0-2]))?))*
+    \s+
+    (?:[0-6]|(?:(?:[0-6]\-[0-6]|\*)(?:\/[0-6])?))(?:,(?:[0-6]|(?:(?:[0-6]\-[0-6]|\*)(?:\/[0-6])?)))*
+  \z/x
+
+  validates :cron, format: { with: CRON_FORMAT }, uniqueness: { scope: :job_definition_id }
+  validate :validate_cron_schedule
+
+  def next(now = Time.now)
+    if 1.month.ago(now).future?
+      Kuroko2.logger.warn("Exceeds the time of criteria #{now}. (Up to 1 month since)")
+      return
+    end
+
+    next_time = Chrono::Iterator.new(self.cron, now: now).next
+    suspend_times = suspend_times(now, next_time)
+
+    if suspend_times.include?(next_time)
+      self.next(next_time)
+    else
+      next_time
+    end
+  end
+
+  def scheduled_times(time_from, time_to)
+    it = Chrono::Iterator.new(cron, now: time_from)
+    scheduled_times = []
+
+    loop do
+      next_time = it.next
+      if next_time <= time_to
+        scheduled_times << next_time
+      else
+        break
+      end
+    end
+
+    scheduled_times
+  end
+
+  def suspend_times(time_from, time_to)
+    if job_definition && job_definition.job_suspend_schedules.present?
+      job_definition.job_suspend_schedules.
+        map { |schedule| schedule.suspend_times(time_from, time_to) }.flatten.uniq
+    else
+      []
+    end
+  end
+
+  private
+
+  def validate_cron_schedule
+    if CRON_FORMAT === cron
+      self.next
+    end
+    nil
+  rescue Chrono::Fields::Base::InvalidField => e
+    errors.add(:cron, "has invalid field: #{e.message}")
+  end
+
+  def self.launch_scheduled_jobs!(time_from, time_to)
+    find_each do |schedule|
+      definition = schedule.job_definition
+      suspend_times = schedule.suspend_times(time_from, time_to)
+
+      schedule.scheduled_times(time_from, time_to).each do |time|
+        if definition.suspended?
+          Kuroko2.logger.info("Skipped suspended \"##{definition.id} #{definition.name}\" that is scheduled at #{I18n.l(time, format: :short)} by `#{schedule.cron}`")
+        elsif suspend_times.include?(time)
+          Kuroko2.logger.info("Skipped schedule suspended \"##{definition.id} #{definition.name}\" that is scheduled at #{I18n.l(time, format: :short)} by `#{schedule.cron}`")
+        else
+          message = "Launched \"##{definition.id} #{definition.name}\" that is scheduled at #{I18n.l(time, format: :short)} by `#{schedule.cron}`"
+          Kuroko2.logger.info(message)
+
+          instance = definition.job_instances.create
+          instance.logs.info(message)
+        end
+      end
+    end
+  end
+end

data/app/models/kuroko2/job_suspend_schedule.rb

@@ -0,0 +1,35 @@
+class Kuroko2::JobSuspendSchedule < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  belongs_to :job_definition
+
+  validates :cron, format: { with: Kuroko2::JobSchedule::CRON_FORMAT }, uniqueness: { scope: :job_definition_id }
+  validate :validate_cron_schedule
+
+  def suspend_times(time_from, time_to)
+    it = Chrono::Iterator.new(cron, now: time_from - 1)
+    suspend_times = []
+
+    loop do
+      next_time = it.next
+      if next_time <= time_to
+        suspend_times << next_time
+      else
+        break
+      end
+    end
+
+    suspend_times
+  end
+
+  private
+
+  def validate_cron_schedule
+    if Kuroko2::JobSchedule::CRON_FORMAT === cron
+      Chrono::Iterator.new(self.cron).next
+    end
+    nil
+  rescue Chrono::Fields::Base::InvalidField => e
+    errors.add(:cron, "has invalid field: #{e.message}")
+  end
+end

data/app/models/kuroko2/log.rb

@@ -0,0 +1,3 @@
+class Kuroko2::Log < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+end

data/app/models/kuroko2/memory_consumption_log.rb

@@ -0,0 +1,49 @@
+class Kuroko2::MemoryConsumptionLog < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  belongs_to :job_instance
+
+  validates :value, presence: true
+
+  # As count becames greater, the interval period will be longer.
+  # First interval will be 1 second and next interval will be 1 second too,
+  # then next interval will be 4 seconds then next interval will be 9 seconds.
+  # Finally, maximum of interval will be 30 minutes.
+  class Interval
+    INITIAL_INTERVAL_PERIOD = 1.second.to_i
+    MAX_INTERVAL_PERIOD = 30.minutes.to_i
+    INCREMENT = 2
+
+    attr_reader :base_time, :count
+
+    # @param [Time] base_time
+    # @param [Integer] count Throttled to be less than 50.
+    def initialize(base_time, count = 0)
+      @base_time = base_time
+      @count = [count, 50].min
+    end
+
+    # @param [Time] now
+    # @return [Boolean]
+    def reached?(now)
+      (now - @base_time) > current_length
+    end
+
+    # @return [MemoryConsumptionLog::Interval]
+    def next
+      self.class.new(Time.at(@base_time.to_i + current_length), @count.succ)
+    end
+
+    private
+
+    # current_length mapping for count (0..50)
+    #
+    # [1, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 196, 225, 256,
+    #  289, 324, 361, 400, 441, 484, 529, 576, 625, 676, 729, 784, 841, 900,
+    #  961, 1024, 1089, 1156, 1225, 1296, 1369, 1444, 1521, 1600, 1681, 1764,
+    #  1800, 1800, 1800, 1800, 1800, 1800, 1800, 1800]
+    def current_length
+      [[@count ** INCREMENT, INITIAL_INTERVAL_PERIOD].max, MAX_INTERVAL_PERIOD].min
+    end
+  end
+end

data/app/models/kuroko2/memory_expectancy.rb

@@ -0,0 +1,21 @@
+class Kuroko2::MemoryExpectancy < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  DEFAULT_VALUE = 0
+
+  belongs_to :job_definition
+
+  validates :expected_value, presence: true
+
+  def memory_consumption_logs
+    Kuroko2::MemoryConsumptionLog.joins(:job_instance).
+      merge(Kuroko2::JobInstance.where(job_definition_id: job_definition_id))
+  end
+
+  # Calculates expected_value with latest consumption logs, then stores it,
+  def calculate!
+    if calculated_value = memory_consumption_logs.maximum(:value)
+      update!(expected_value: calculated_value)
+    end
+  end
+end

data/app/models/kuroko2/process_signal.rb

@@ -0,0 +1,14 @@
+class Kuroko2::ProcessSignal < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  scope :unstarted, -> { where(started_at: nil) }
+  scope :on, ->(hostname) { where(hostname: hostname) }
+
+  def self.poll(hostname)
+    self.transaction do
+      unstarted.on(hostname).lock.take.tap do |signal|
+        signal.touch(:started_at) if signal
+      end
+    end
+  end
+end

data/app/models/kuroko2/star.rb

@@ -0,0 +1,6 @@
+class Kuroko2::Star < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  belongs_to :user
+  belongs_to :job_definition
+end

data/app/models/kuroko2/tag.rb

@@ -0,0 +1,8 @@
+class Kuroko2::Tag < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  has_many :job_definition_tags
+  has_many :job_definitions, through: :job_definition_tags
+
+  validates :name, length: { maximum: 100 }
+end

data/app/models/kuroko2/tick.rb

@@ -0,0 +1,12 @@
+class Kuroko2::Tick < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  def self.fetch_then_update(now)
+    tick = self.first_or_create
+    last = tick.at || now
+
+    tick.update_column(:at, now)
+    last
+  end
+
+end

data/app/models/kuroko2/token.rb

@@ -0,0 +1,101 @@
+class Kuroko2::Token < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  WORKING  = 0
+  FINISHED = 1
+  FAILURE  = 2
+  WAITING  = 3
+  CRITICAL = 9
+
+  STATUSES = {
+    WORKING  => :working,
+    FINISHED => :finished,
+    FAILURE  => :failure,
+    CRITICAL => :critical,
+    WAITING  => :waiting,
+  }.freeze
+
+  serialize :context, JSON
+
+  belongs_to :job_definition
+  belongs_to :job_instance
+
+  has_many :children, class_name: 'Token', foreign_key: 'parent_id', dependent: :destroy
+  belongs_to :parent, class_name: 'Token', optional: true
+
+  has_one :execution
+
+  before_create :set_default_values
+
+  scope :processable, -> { where(status: [WORKING, WAITING])}
+  scope :working, -> { where(status: WORKING) }
+  scope :finished, -> { where(status: FINISHED) }
+  scope :waiting, -> { where(status: WAITING) }
+
+  def working?
+    status == WORKING
+  end
+
+  def failure?
+    status == FAILURE
+  end
+
+  def finished?
+    status == FINISHED
+  end
+
+  def critical?
+    status == CRITICAL
+  end
+
+  def waiting?
+    status == WAITING
+  end
+
+  def mark_as_failure
+    self.status = FAILURE
+  end
+
+  def mark_as_critical(error)
+    self.status  = CRITICAL
+    self.message = error.message
+  end
+
+  def mark_as_finished
+    self.status = FINISHED
+  end
+
+  def mark_as_working
+    self.status = WORKING
+  end
+
+  def mark_as_waiting
+    self.status = WAITING
+  end
+
+  def status_name
+    STATUSES[status].to_s
+  end
+
+  def cancelable?
+    case status
+    when WORKING, WAITING
+      children.many? && children.all? do |child|
+        child.status == FINISHED || child.cancelable?
+      end
+    when FAILURE
+      true
+    else
+      false
+    end
+  end
+
+  private
+
+  def set_default_values
+    self.uuid    ||= SecureRandom.uuid
+    self.message ||= ''
+    self.context ||= {}
+    self.status  ||= WORKING
+  end
+end

data/app/models/kuroko2/user.rb

@@ -0,0 +1,48 @@
+class Kuroko2::User < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  GRAVATAR_URL = '//www.gravatar.com/avatar/%s?s=90&d=mm'
+  GOOGLE_OAUTH2_PROVIDER = 'google_oauth2'
+  GROUP_PROVIDER = 'group_mail'
+
+  paginates_per 100
+
+  scope :active, -> { where(suspended_at: nil) }
+  scope :with, -> (ids) { where(id: ids) }
+  scope :group_user, -> { where(provider: GROUP_PROVIDER) }
+
+  has_many :stars
+  has_many :job_definitions, through: :stars
+
+  has_many :admin_assignments, dependent: :restrict_with_error
+  has_many :assigned_job_definitions, through: :admin_assignments, source: :job_definition
+
+  validates :name, uniqueness: { case_sensitive: false} , presence: true
+  validates :email, uniqueness: { case_sensitive: false}, presence: true
+
+  before_create :set_gravatar_image
+
+  def self.find_or_create_user(uid, attributes)
+    find_or_create_by(uid: uid) do |user|
+      user.name       = attributes[:name]
+      user.email      = attributes[:email]
+      user.first_name = attributes[:first_name]
+      user.last_name  = attributes[:last_name]
+    end
+  end
+
+  def google_account?
+    self.provider == GOOGLE_OAUTH2_PROVIDER
+  end
+
+  private
+
+  def set_gravatar_image
+    self.image = gravatar_url(self.email)
+  end
+
+  def gravatar_url(email)
+    GRAVATAR_URL % Digest::MD5::hexdigest(email.strip.downcase)
+  end
+
+end

data/app/models/kuroko2/worker.rb

@@ -0,0 +1,12 @@
+class Kuroko2::Worker < Kuroko2::ApplicationRecord
+  include Kuroko2::TableNameCustomizable
+
+  belongs_to :execution, optional: true
+
+  scope :on, -> (hostname) { where(hostname: hostname) }
+  scope :ordered, -> { order(:hostname, :worker_id) }
+
+  def self.executing(id)
+    where(execution_id: id).take
+  end
+end