little_monster 0.1.0

data/lib/little_monster/core/counters.rb

@@ -0,0 +1,50 @@
+require_relative '../core/loggable'
+require_relative '../core/api'
+
+module LittleMonster::Core::Counters
+  def increase_counter(counter_name, unique_id, type, output = '')
+    begin
+      resp = LittleMonster::Core::API.put(
+        "/jobs/#{job_id}/counters/#{counter_name}",
+        { body: { type: type, unique_id: unique_id, output: output } },
+        critical: true
+      )
+    rescue LittleMonster::APIUnreachableError => e
+      logger.error "Could not increase counter #{counter_name}, Api unreachable"
+      raise e
+    end
+    raise DuplicatedCounterError if resp.code == 412
+    true
+  end
+
+  def counter(counter_name)
+    resp = LittleMonster::Core::API.get("/jobs/#{job_id}/counters/#{counter_name}", {}, critical: true)
+    raise MissedCounterError if resp.code == 404
+    resp.body
+  end
+
+  def counter_endpoint(name)
+    "#{LittleMonster.api_url.chomp('/')}/jobs/#{job_id}/counters/#{name}"
+  end
+
+  def init_counters(*counter_names)
+    counter_names.each do |counter_name|
+      resource = "/jobs/#{job_id}/counters"
+      values = { body: { name: counter_name } }
+      begin
+        res = LittleMonster::Core::API.post(resource, values, critical: true)
+        raise MissedCounterError, "Could not post to #{resource}" if !res.success? && res.code != 409 # counter already exists
+      rescue LittleMonster::APIUnreachableError => e
+        logger.error "Could not init counter #{resource} with #{values} , Api unreachable"
+        raise e
+      end
+    end
+    true
+  end
+
+  class MissedCounterError < StandardError
+  end
+
+  class DuplicatedCounterError < StandardError
+  end
+end

data/lib/little_monster/core/errors/api_unreachable_error.rb

@@ -0,0 +1,4 @@
+module LittleMonster::Core
+  class APIUnreachableError < StandardError
+  end
+end

data/lib/little_monster/core/errors/callback_failed_error.rb

@@ -0,0 +1,4 @@
+module LittleMonster::Core
+  class CallbackFailedError < StandardError
+  end
+end

data/lib/little_monster/core/errors/cancel_error.rb

@@ -0,0 +1,4 @@
+module LittleMonster::Core
+  class CancelError < TaskError
+  end
+end

data/lib/little_monster/core/errors/fatal_task_error.rb

@@ -0,0 +1,4 @@
+module LittleMonster::Core
+  class FatalTaskError < TaskError
+  end
+end

data/lib/little_monster/core/errors/job_already_locked_error.rb

@@ -0,0 +1,4 @@
+module LittleMonster::Core
+  class JobAlreadyLockedError < StandardError
+  end
+end

data/lib/little_monster/core/errors/job_not_found_error.rb

@@ -0,0 +1,15 @@
+module LittleMonster::Core
+  class JobNotFoundError < StandardError
+    def initialize(job_id)
+      params = {
+        body: {
+          status: 'error'
+        }
+      }
+      LittleMonster::API.put "/jobs/#{job_id}", params,
+                             retries: LittleMonster.job_requests_retries,
+                             retry_wait: LittleMonster.job_requests_retry_wait,
+                             critical: true
+    end
+  end
+end

data/lib/little_monster/core/errors/job_retry_error.rb

@@ -0,0 +1,4 @@
+module LittleMonster::Core
+  class JobRetryError < StandardError
+  end
+end

data/lib/little_monster/core/errors/max_retries_error.rb

@@ -0,0 +1,4 @@
+module LittleMonster::Core
+  class MaxRetriesError < StandardError
+  end
+end

data/lib/little_monster/core/errors/task_error.rb

@@ -0,0 +1,4 @@
+module LittleMonster::Core
+  class TaskError < StandardError
+  end
+end

data/lib/little_monster/core/job.rb

@@ -0,0 +1,188 @@
+module LittleMonster::Core
+  class Job
+    include Loggable
+
+    ENDED_STATUS = %i(success error cancelled).freeze
+    CALLBACKS = %i(on_success on_error on_cancel).freeze
+
+    class << self
+      def task_list(*tasks)
+        @tasks = *tasks
+      end
+
+      def retries(value)
+        @max_retries = value
+      end
+
+      def task_class_for(task_name)
+        "#{to_s.underscore}/#{task_name}".camelcase.constantize
+      end
+
+      def max_retries
+        @max_retries ||= -1
+      end
+
+      def mock!
+        @@mock = true
+      end
+
+      def tasks
+        @tasks ||= []
+      end
+
+      def mock?
+        @@mock ||= false
+      end
+    end
+
+    attr_accessor :id
+    attr_accessor :tags
+    attr_accessor :status
+
+    attr_accessor :retries
+    attr_accessor :current_action
+    attr_accessor :data
+
+    attr_reader :orchrestator
+
+    def initialize(options = {})
+      @id = options.fetch(:id, nil)
+      @tags = (options[:tags] || {}).freeze
+
+      @retries = options[:retries] || 0
+
+      @current_action = options.fetch(:current_action, self.class.tasks.first)
+
+      @data = if options[:data]
+                Data.new(self, options[:data])
+              else
+                Data.new(self)
+              end
+
+      @status = options.fetch(:status, :pending)
+
+      @orchrestator = Job::Orchrestator.new(self)
+
+      if mock?
+        @runned_tasks = {}
+        self.class.send :attr_reader, :runned_tasks
+      end
+
+      logger.default_tags = tags.merge(
+        id: @id,
+        job: self.class.to_s,
+        retry: @retries
+      )
+
+      logger.info "[type:start_job] Starting job with data: #{data.to_h[:outputs]}"
+    end
+
+    def run
+      @orchrestator.run
+    end
+
+    def notify_status(options = {})
+      params = { body: { status: @status } }
+      params[:body].merge!(options)
+
+      notify_job params, retries: LittleMonster.job_requests_retries,
+                         retry_wait: LittleMonster.job_requests_retry_wait
+    end
+
+    def notify_task(status, options = {})
+      params = { body: { tasks: [{ name: @current_action, status: status }] } }
+      params[:body][:data] = options[:data] if options[:data]
+
+      params[:body][:tasks].first.merge!(options.except(:data))
+
+      notify_job params, retries: LittleMonster.task_requests_retries,
+                         retry_wait: LittleMonster.task_requests_retry_wait
+    end
+
+    def notify_callback(status, options = {})
+      return true unless should_request?
+      params = { body: { name: @current_action, status: status } }
+      params[:body].merge!(options)
+
+      resp = LittleMonster::API.put "/jobs/#{id}/callbacks/#{@current_action}", params,
+                                    retries: LittleMonster.task_requests_retries,
+                                    retry_wait: LittleMonster.task_requests_retry_wait
+      resp.success?
+    end
+
+    def notify_job(params = {}, options = {})
+      return true unless should_request?
+      options[:critical] = true
+
+      params[:body][:data] = params[:body][:data].to_h if params[:body][:data]
+
+      resp = LittleMonster::API.put "/jobs/#{id}", params, options
+      resp.success?
+    end
+
+    def is_cancelled?
+      return false unless should_request?
+      resp = LittleMonster::API.get "/jobs/#{id}"
+
+      if resp.success?
+        resp.body[:cancel]
+      else
+        false
+      end
+    end
+
+    def task_class_for(task_name)
+      self.class.task_class_for task_name
+    end
+
+    def max_retries
+      self.class.max_retries
+    end
+
+    def retry?
+      max_retries == -1 || max_retries > @retries
+    end
+
+    def callback_to_run
+      case @status
+      when :success
+        :on_success
+      when :error
+        :on_error
+      when :cancelled
+        :on_cancel
+      end
+    end
+
+    # returns the tasks that will be runned for this instance
+    def tasks_to_run
+      return [] if callback_running?
+      task_index = self.class.tasks.find_index(@current_action)
+
+      return [] if task_index.nil?
+      self.class.tasks.slice(task_index..-1)
+    end
+
+    def callback_running?
+      return false if @current_action.nil? || self.class.tasks.include?(@current_action)
+      CALLBACKS.include? @current_action
+    end
+
+    def ended_status?
+      Job::ENDED_STATUS.include? @status
+    end
+
+    def mock?
+      self.class.mock?
+    end
+
+    def should_request?
+      !(mock? || LittleMonster.disable_requests?)
+    end
+
+    # callbacks definition
+    def on_error ; end
+    def on_success ; end
+    def on_cancel ; end
+  end
+end

data/lib/little_monster/core/job_data.rb

@@ -0,0 +1,47 @@
+module LittleMonster::Core
+  class Job::Data
+    def initialize(job, input = {})
+      @outputs = input.fetch(:outputs, {})
+      @key_owners = input.fetch(:owners, {})
+      @job = job
+    end
+
+    def ==(other)
+      return false unless is_valid?(other) && other.length == length
+      @outputs.each { |k, v| return false unless other[k.to_sym] == v }
+      true
+    end
+
+    def [](output_key)
+      @outputs[output_key.to_sym]
+    end
+
+    def []=(output_key, value)
+      raise KeyError, "The key #{output_key} already exists" if @outputs.include? output_key.to_sym
+      @outputs[output_key.to_sym] = value
+
+      owner = @job.current_action.to_sym
+      @key_owners[owner] = [] unless @key_owners[owner].is_a? Array
+      @key_owners[owner] << output_key.to_sym
+    end
+
+    def to_json
+      MultiJson.dump(to_h)
+    end
+
+    def to_h
+      return {} if @outputs.empty?
+      { outputs: @outputs, owners: @key_owners }
+    end
+
+    def length
+      @outputs.length
+    end
+
+    private
+
+    def is_valid?(other)
+      other.instance_of?(Job::Data) || other.instance_of?(Hash)
+    end
+  end
+end

data/lib/little_monster/core/job_factory.rb

@@ -0,0 +1,139 @@
+module LittleMonster::Core
+  class Job::Factory
+    include Loggable
+
+    def initialize(message = {})
+      @id = message[:id]
+      @name = message[:name]
+
+      # it converts tags from array of hashes to a single hash
+      @tags = Hash[message.fetch(:tags, []).map { |h| [h.keys.first, h.values.first] }].freeze
+
+      logger.default_tags = @tags.merge(id: @id, name: @name)
+
+      @api_attributes = fetch_attributes.freeze
+
+      # this gets saved for development run and debugging purposes
+      @input_data = message[:data]
+
+      begin
+        @job_class = @name.to_s.camelcase.constantize
+      rescue NameError
+        raise JobNotFoundError.new(@id), "[type:error] job [name:#{@name}] does not exists"
+      end
+    end
+
+    def build
+      if discard?
+        logger.info "[type:discard] discarding job with [status:#{(@api_attributes || {}).fetch(:status, 'nil')}]"
+        return
+      end
+
+      unless LittleMonster.disable_requests?
+        notify_job_task_list
+        notify_job_max_retries
+      end
+
+      @job_class.new job_attributes
+    end
+
+    def notify_job_task_list
+      return true unless @api_attributes[:tasks].blank?
+
+      params = {
+        body: {
+          tasks: @job_class.tasks.each_with_index.map { |task, index| { name: task, order: index } }
+        }
+      }
+
+      res = LittleMonster::API.post "/jobs/#{@id}/tasks", params, retries: LittleMonster.job_requests_retries,
+                                                                  retry_wait: LittleMonster.job_requests_retry_wait,
+                                                                  critical: true
+      res.success?
+    end
+
+    def notify_job_max_retries
+      return true unless @api_attributes[:max_retries].blank?
+
+      params = {
+        body: { max_retries: @job_class.max_retries }
+      }
+
+      res = LittleMonster::API.put "/jobs/#{@id}", params, retries: LittleMonster.job_requests_retries,
+                                                           retry_wait: LittleMonster.job_requests_retry_wait
+      res.success?
+    end
+
+    def fetch_attributes
+      return {} if LittleMonster.disable_requests?
+      resp = API.get "/jobs/#{@id}", {}, retries: LittleMonster.job_requests_retries,
+                                         retry_wait: LittleMonster.job_requests_retry_wait,
+                                         critical: true
+
+      resp.success? ? resp.body : nil
+    end
+
+    def calculate_status
+      return :pending if @api_attributes[:tasks].blank?
+
+      @api_attributes[:tasks].sort_by! { |task| task[:order] }.each do |task|
+        return task[:status].to_sym if task[:status].to_sym != :success
+      end
+
+      # check if any callback failed
+      @api_attributes.fetch(:callbacks, []).each do |callback|
+        return :error if callback[:status].to_sym == :error
+      end
+
+      :success
+    end
+
+    def find_current_action_and_retries
+      return [@job_class.tasks.first, 0] if @api_attributes[:tasks].blank?
+
+      # callbacks and tasks both have name, retries and status
+      # that means we can search through them with the same block
+
+      search_array = if @api_attributes.fetch(:callbacks, []).blank?
+                       # callbacks have not run yet, so we look for tasks
+                       @api_attributes[:tasks].sort_by { |task| task[:order] }
+                     else
+                       @api_attributes[:callbacks]
+                     end
+
+      current = search_array.find do |x|
+        !Job::ENDED_STATUS.include? x[:status].to_sym
+      end
+      return nil unless current
+
+      [current[:name].to_sym, current[:retries]]
+    end
+
+    def job_attributes
+      data = if !@api_attributes[:data].nil?
+               @api_attributes[:data]
+             else
+               @input_data
+             end
+
+      attributes = {
+        id: @id,
+        data: data,
+        tags: @tags
+      }
+
+      return attributes if LittleMonster.disable_requests?
+
+      status = calculate_status
+      current_action, retries = find_current_action_and_retries
+
+      attributes.merge(status: status,
+                       current_action: current_action,
+                       retries: retries)
+    end
+
+    def discard?
+      @api_attributes.nil? || Job::ENDED_STATUS.include?(@api_attributes.fetch(:status, :pending).to_sym)
+    end
+  end
+end