little_monster 0.1.0

data/lib/little_monster/core/job_orchrestator.rb

@@ -0,0 +1,194 @@
+# TODO : don't send data on callback fail
+module LittleMonster::Core
+  class Job::Orchrestator
+    attr_reader :logger
+    attr_reader :job
+
+    def initialize(job)
+      @job = job
+      @logger = @job.logger
+    end
+
+    def run
+      # notifies status as running and then restores old_status if it is an ending status
+      last_status = @job.status
+      @job.status = :running
+      @job.notify_status
+
+      if Job::ENDED_STATUS.include? last_status
+        @job.status = last_status
+      else
+        run_tasks
+
+        logger.default_tags.delete(:current_task)
+        # reset retries so retries don't mix between tasks and callbacks
+        @job.retries = 0
+      end
+
+      run_callback
+      logger.info "[type:job_finish] [status:#{@job.status}] data: #{@job.data.to_h[:outputs]}"
+    ensure
+      options = {}
+      options[:data] = @job.data if @job.ended_status?
+      @job.notify_status options
+    end
+
+    def run_tasks
+      @job.tasks_to_run.each do |task_name|
+        @job.current_action = task_name
+        @job.notify_task :running
+
+        logger.default_tags[:current_task] = @job.current_action
+        logger.info "[type:start_task] data: #{@job.data.to_h[:outputs]}"
+
+        begin
+          raise LittleMonster::CancelError if @job.is_cancelled?
+
+          task = build_task(task_name)
+          task.run
+
+          # data is sent only on task success
+          @job.notify_task :success, data: @job.data
+
+          logger.info "[type:finish_task] [status:success] data: #{@job.data.to_h[:outputs]}"
+
+          if @job.mock?
+            @job.runned_tasks[task_name] = {}
+            @job.runned_tasks[task_name][:instance] = task
+            @job.runned_tasks[task_name][:data] = @job.data.to_h[:outputs].to_h.dup
+          end
+        rescue APIUnreachableError => e
+          logger.error "[type:api_unreachable] [message:#{e.message}]"
+          raise e
+        rescue CancelError => e
+          logger.info '[type:cancel] job was cancelled'
+          cancel
+          return
+        rescue StandardError => e
+          logger.debug "[type:standard_error] an error was catched with [message:#{e.message}]"
+          task.error e unless e.is_a? NameError
+          handle_error e
+          return
+        end
+
+        @job.retries = 0 # Hago esto para que despues de succesful un task resete retries
+      end
+
+      @job.current_action = nil
+      @job.status = :success
+    end
+
+    def run_callback
+      @job.current_action = @job.callback_to_run
+
+      return if @job.current_action.nil?
+
+      logger.default_tags[:callback] = @job.current_action
+      @job.notify_callback :running
+
+      logger.info "[type:start_callback] data: #{@job.data.to_h[:outputs]}"
+      begin
+        logger.default_tags[:type] = 'callback_log'
+        @job.public_send(@job.current_action)
+      ensure
+        logger.default_tags.delete(:type)
+      end
+      logger.info "[type:finish_callback] [status:success] data: #{@job.data.to_h[:outputs]}"
+
+      @job.notify_callback :success
+
+      @job.current_action = nil
+      @job.retries = 0
+      logger.default_tags.delete(:callback)
+    rescue APIUnreachableError => e
+      logger.error "[type:api_unreachable] [message:#{e.message}]"
+      raise e
+    rescue StandardError => e
+      logger.debug "[type:standard_error] an error was catched with [message:#{e.message}]"
+      handle_error e
+    end
+
+    def build_task(task_symbol)
+      task = @job.task_class_for(task_symbol).new(@job.data)
+      task.send(:set_default_values, @job.data, @job.id, logger, @job.method(:is_cancelled?))
+      task
+    end
+
+    def cancel
+      logger.debug 'notifiying cancel...'
+
+      @job.notify_task :cancelled
+      logger.info "[type:finish_task] [status:cancelled] data: #{@job.data.to_h[:outputs]}"
+
+      @job.status = :cancelled
+    end
+
+    # Methods that work both on tasks and callbacks
+
+    def abort_job(_e)
+      logger.debug 'notifiying abort...'
+
+      if @job.callback_running?
+        logger.info "[type:finish_callback] [status:error] data: #{@job.data.to_h[:outputs]}"
+        @job.notify_callback :error
+
+        # if callback is not on_error, raise exception to run on_error
+        if @job.current_action != :on_error
+          # set status on pending because we are sending the job back to the queue
+          @job.status = :pending
+          raise CallbackFailedError, '[type:callback_fail_error]'
+        end
+      else
+        @job.notify_task :error
+        logger.info "[type:finish_task] [status:error] data: #{@job.data.to_h[:outputs]}"
+      end
+
+      @job.status = :error
+    end
+
+    def handle_error(e)
+      raise e if LittleMonster.env.development?
+      logger.error "[type:error] [error_type:#{e.class}][message:#{e.message}] \n #{e.backtrace.to_a.join("\n\t")}"
+
+      if e.is_a?(FatalTaskError) || e.is_a?(NameError)
+        logger.debug 'error is fatal, aborting run'
+        return abort_job(e)
+      end
+
+      do_retry
+    end
+
+    def do_retry
+      if @job.retry?
+        logger.debug "Retry ##{@job.retries} of #{@job.max_retries}"
+
+        @job.retries += 1
+
+        logger.debug 'notifiying retry'
+        if @job.callback_running?
+          @job.notify_callback :pending, retries: @job.retries
+          logger.info '[type:callback_retry]'
+        else
+          @job.notify_task :pending, retries: @job.retries
+          logger.info '[type:task_retry]'
+        end
+
+        @job.status = :pending
+
+        logger.info "[type:job_retry] data: #{@job.data.to_h[:outputs]}"
+        raise JobRetryError, "doing retry #{@job.retries} of #{@job.max_retries}"
+      else
+        logger.debug 'job has reached max retries'
+
+        if @job.callback_running?
+          logger.info '[type:callback_max_retries]'
+        else
+          logger.info '[type:task_max_retries]'
+        end
+
+        logger.info "[type:job_max_retries] [retries:#{@job.max_retries}]"
+        abort_job(MaxRetriesError.new)
+      end
+    end
+  end
+end

data/lib/little_monster/core/loggable.rb

@@ -0,0 +1,7 @@
+module LittleMonster::Core
+  module Loggable
+    def logger
+      @logger ||= TaggedLogger.new
+    end
+  end
+end

data/lib/little_monster/core/runner.rb

@@ -0,0 +1,39 @@
+module LittleMonster::Core
+  class Runner
+    include Loggable
+
+    def initialize(params)
+      @params = params
+
+      @heartbeat_task = Concurrent::TimerTask.new(execution_interval: LittleMonster.heartbeat_execution_interval) do
+        send_heartbeat!
+      end
+    end
+
+    def run
+      send_heartbeat!
+
+      @heartbeat_task.execute unless LittleMonster.disable_requests?
+
+      job = LittleMonster::Job::Factory.new(@params).build
+      job.run unless job.nil?
+    rescue JobNotFoundError => e
+      logger.error "[id:#{@params[:id]}][type:job_not_found] [message:#{e.message}] \n #{e.backtrace.to_a.join("\n\t")}"
+    ensure
+      @heartbeat_task.shutdown
+    end
+
+    def send_heartbeat!
+      return if LittleMonster.disable_requests?
+
+      res = LittleMonster::API.put "/jobs/#{@params[:id]}/worker", body: {
+        ip: Socket.gethostname,
+        host: Socket.gethostname,
+        pid: Process.pid
+      }
+
+      raise LittleMonster::JobAlreadyLockedError, "job [id:#{@params[:id]}] is already locked, discarding" if res.code == 401
+      res.success?
+    end
+  end
+end

data/lib/little_monster/core/tagged_logger.rb

@@ -0,0 +1,66 @@
+module LittleMonster::Core
+  class TaggedLogger
+    attr_accessor :parent_logger
+    attr_reader :tags
+
+    LEVELS = [:unknown, :fatal, :error, :warn, :info, :debug].freeze
+
+    def self.tags_to_string(hash)
+      hash.map { |k, v| "[#{k}:#{v}]" }.join
+    end
+
+    def initialize
+      @tags = Hash.new({})
+      @parent_logger = nil
+    end
+
+    def method_missing(method, *args, &block)
+      if method.to_s.ends_with? 'tags='
+        tag_key = method.to_s.split('_').first.to_sym
+        return public_send('tags_for', tag_key, *args) if LEVELS.include? tag_key
+      end
+
+      if method.to_s.ends_with? 'tags'
+        tag_key = method.to_s.split('_').first.to_sym
+        return @tags[tag_key] if LEVELS.include? tag_key
+      end
+
+      if LEVELS.include? method.to_sym
+        return LittleMonster.logger.public_send method, tag_message(method.to_sym, *args)
+      end
+
+      super method, *args, &block
+    end
+
+    def tags_for(key, t = {})
+      @tags[key] = t
+    end
+
+    def default_tags
+      @tags[:default]
+    end
+
+    def default_tags=(t)
+      tags_for(:default, t)
+    end
+
+    def tag_message(level, message = '')
+      prefix_string =  tags_to_string @tags[:default].merge(@tags[level])
+      prefix_string << ' -- ' unless prefix_string.blank?
+
+      unless @parent_logger.nil?
+        prefix_string = @parent_logger.tag_message level, prefix_string
+      end
+
+      [prefix_string, message].join
+    end
+
+    def log_tags(level, tags_hash)
+      public_send(level, tags_to_string(tags_hash))
+    end
+
+    def tags_to_string(hash)
+      self.class.tags_to_string hash
+    end
+  end
+end

data/lib/little_monster/core/task.rb

@@ -0,0 +1,41 @@
+module LittleMonster::Core
+  class Task
+    include Loggable
+
+    attr_reader :data
+    attr_reader :job_id
+
+    def initialize(data, job_id = nil)
+      @data = data
+      @job_id = job_id
+    end
+
+    def run
+      raise NotImplementedError, 'You must implement the run method'
+    end
+
+    def on_error(error)
+    end
+
+    def error(e)
+      logger.error e
+      on_error e
+    end
+
+    def is_cancelled!
+      is_cancelled = false
+      is_cancelled = @cancelled_callback.call unless @cancelled_callback.nil?
+      raise CancelError if is_cancelled
+    end
+
+    private
+
+    def set_default_values(data, job_id = nil, job_logger = nil, cancelled_callback = nil)
+      @cancelled_callback = cancelled_callback
+      @job_id = job_id
+      @data = data
+      logger.parent_logger = job_logger if job_logger
+      logger.default_tags.merge!(type: 'task_log')
+    end
+  end
+end

data/lib/little_monster/generators/cli.rb

@@ -0,0 +1,75 @@
+require 'thor'
+require_relative './conf_gen'
+require_relative './generate'
+
+module LittleMonster
+  class Cli < Thor
+    desc 'show version', 'version'
+    map %w(-v --version) => :version
+
+    def version
+      say LittleMonster::VERSION
+    end
+
+    desc 'exec <job>', 'runs a job'
+    option :message,
+           type: :hash,
+           aliases: :m,
+           default: {}
+
+    method_option :message,
+                  aliases: '-m',
+                  type: :string,
+                  default: '{}',
+                  desc: 'Message that will be send as parameter (must be a JSON format)'
+
+    method_option :record_mode,
+                  aliases: '-r',
+                  type: :string,
+                  enum: %w(none new reload),
+                  default: 'none',
+                  desc: 'Recording mocks mode  none|new|reload',
+                  banner: 'Recording type could be none,new or reload on default assume none'
+
+    def exec(job)
+      ENV['LITTLE_MONSTER_ENV'] = options[:environment]
+      require_relative "#{Dir.pwd}/config/application.rb"
+      require 'webmock'
+      require 'vcr'
+
+      msg = MultiJson.load(options[:message], symbolize_keys: true)
+      params = { data: { outputs: msg }, name: job }
+      vcr_mode = { 'none' => :none,
+                   'new' => :new_episodes,
+                   'reload' => :all }.fetch(options[:record_mode], :none)
+      VCR.configure do |config|
+        config.cassette_library_dir = 'mocks/vcr_cassettes'
+        config.hook_into :webmock # or :fakeweb
+      end
+
+      VCR.use_cassette(job.to_s, record: vcr_mode) do
+        LittleMonster::Runner.new(params).run
+      end
+    end
+
+    desc 'start', 'starts the little monster worker'
+    option :daemonize,
+           type: :boolean,
+           default: false,
+           aliases: :d
+
+    def start
+      require_relative "#{Dir.pwd}/config/application.rb"
+
+      toiler_args = ['-C', "#{Dir.pwd}/config/toiler.yml"]
+      toiler_args += ['-d', '-L', 'log/little_monster.log'] if options[:daemonize]
+      Toiler::CLI.instance.run(toiler_args)
+    end
+
+    register(LittleMonster::ConfGen, 'init', 'init', 'Creates new Little Monster Schema app')
+    register(LittleMonster::Generate,
+             'generate',
+             'generate <job_name> <task_list>...',
+             'Creates a job with his respective tasks.')
+  end
+end

data/lib/little_monster/generators/conf_gen.rb

@@ -0,0 +1,28 @@
+require 'thor'
+require 'active_support/core_ext/string'
+
+module LittleMonster
+  class ConfGen < Thor::Group
+    include Thor::Actions
+
+    def self.source_root
+      File.dirname(__FILE__)
+    end
+
+    def create_conf_files
+      directory('./templates/config', 'config')
+    end
+
+    def create_lib_files
+      directory('./templates/lib', 'lib')
+    end
+
+    def create_log_files
+      directory('./templates/log', 'log')
+    end
+
+    def create_specs_files
+      template 'templates/spec_helper_temp.erb', 'spec/spec_helper.rb'
+    end
+  end
+end