qyu 1.0.0

data/lib/qyu/models/job.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+module Qyu
+  class Job
+    attr_reader :descriptor, :payload, :id, :created_at, :updated_at
+
+    def self.create(workflow:, payload:)
+      workflow = Workflow.find_by(name: workflow) if workflow.is_a?(String)
+      id = persist(workflow, payload)
+      time = Time.now
+      new(id, workflow, payload, time, time)
+    end
+
+    def self.find(id)
+      job_attrs = Qyu.store.find_job(id)
+      new(id, job_attrs['workflow'], job_attrs['payload'],
+          job_attrs['created_at'], job_attrs['updated_at'])
+    end
+
+    def self.select(limit: 30, offset: 0, order: :asc)
+      job_records = Qyu.store.select_jobs(limit, offset, order)
+      job_records.map do |record|
+        new(record['id'], record['workflow'], record['payload'],
+            record['created_at'], record['updated_at'])
+      end
+    end
+
+    def self.count
+      Qyu.store.count_jobs
+    end
+
+    def self.delete(id)
+      Qyu.store.delete_job(id)
+    end
+
+    def self.clear_completed
+      Qyu.store.clear_completed_jobs
+    end
+
+    def start
+      descriptor['starts'].each do |task_name|
+        create_task(nil, task_name, payload)
+      end
+    end
+
+    def queue_name(task_name)
+      descriptor['tasks'][task_name]['queue']
+    end
+
+    def next_task_names(src_task_name)
+      {
+        'without_params' => descriptor['tasks'][src_task_name]['starts'],
+        'with_params' => descriptor['tasks'][src_task_name]['starts_with_params']
+      }
+    end
+
+    def tasks_to_wait_for(task)
+      descriptor['tasks'][task.name]['waits_for'].keys
+    end
+
+    def sync_condition(task, task_name)
+      descriptor['tasks'][task.name]['waits_for'][task_name]['condition']
+    end
+
+    def create_task(parent_task, task_name, payload)
+      parent_task_id = parent_task.nil? ? nil : parent_task.id
+      Qyu.logger.debug "Task (ID=#{parent_task_id}) created a new task"
+      Qyu::Task.create(
+        queue_name: queue_name(task_name),
+        attributes: {
+                      'name' => task_name,
+                      'parent_task_id' => parent_task_id,
+                      'job_id' => id,
+                      'payload' => task_payload(payload, task_name)
+                    })
+    end
+
+    def create_next_tasks(parent_task, payload)
+      Qyu.logger.debug "Creating next tasks for task (ID=#{parent_task.id})"
+      next_tasks = next_task_names(parent_task.name)
+      Qyu.logger.debug "Next task names: #{next_tasks}"
+
+      next_tasks['without_params']&.each do |next_task_name|
+        create_task(parent_task, next_task_name, payload)
+      end
+
+      next_tasks['with_params']&.each do |next_task_name, params|
+        updated_payload = payload.dup
+        params.each do |param_name, value_eqs|
+          f = value_eqs.keys[0]
+          x = value_eqs.values[0]
+          updated_payload[param_name] = calc_func_x(parent_task, f, x)
+        end
+        create_task(parent_task, next_task_name, updated_payload)
+      end
+    end
+
+    def find_task_ids_by_name(task_name)
+      Qyu.store.find_task_ids_by_job_id_and_name(id, task_name)
+    end
+
+    def find_task_ids_by_name_and_ancestor_task_id(task_name, ancestor_task_id)
+      ancestor_task_name = Qyu.store.find_task(ancestor_task_id)['name']
+      tasks_path = [task_name]
+      key_idx = 0
+
+      while tasks_path[-1] != ancestor_task_name
+        found_task = descriptor['tasks'].detect do |_, desc|
+          all_task_names = []
+          all_task_names.concat(desc['starts'] || [])
+          all_task_names.concat((desc['starts_with_params'] || {}).keys)
+          all_task_names.concat(desc['starts_manually'] || [])
+          all_task_names.include?(tasks_path[-1])
+        end
+        tasks_path << found_task[key_idx] if found_task
+      end
+
+      tasks_topdown_path = tasks_path.reverse
+      # remove topmost task (ancestor_task) from the path
+      tasks_topdown_path.shift
+
+      # traverse task tree from top down, and find the <task_name> "descendants" of <ancestor_task>
+      parent_task_ids = [ancestor_task_id]
+      tasks_topdown_path.each do |t_name|
+        parent_task_ids = Qyu.store.find_task_ids_by_job_id_name_and_parent_task_ids(id, t_name, parent_task_ids)
+      end
+      parent_task_ids
+    end
+
+    def task_status_counts
+      Qyu.store.task_status_counts(id)
+    end
+
+    def [](attribute)
+      public_send(attribute)
+    end
+
+    private_class_method :new
+
+    private
+
+    def initialize(id, workflow, payload, created_at = nil, updated_at = nil)
+      @workflow = workflow
+      @descriptor = @workflow['descriptor']
+      @payload = payload
+      @id = id
+      @created_at = created_at
+      @updated_at = updated_at
+    end
+
+    def self.persist(workflow, payload)
+      workflow = Qyu::Workflow.find_by(name: workflow) if workflow.is_a?(String)
+      Qyu.store.persist_job(workflow, payload)
+    end
+
+    def calc_func_x(task, func, x)
+      if func == 'count'
+        find_task_ids_by_name_and_ancestor_task_id(x, task.id).count
+      else
+        fail Qyu::Errors::NotImplementedError
+      end
+    end
+
+    def task_payload(payload, task_name)
+      shared_payload = payload.dup.reject { |k, _v| task_name?(k) }
+      shared_payload.merge!(payload[task_name]) if payload[task_name].is_a?(Hash)
+      shared_payload
+    end
+
+    def task_name?(string)
+      descriptor['tasks'].keys.include?(string)
+    end
+  end
+end

data/lib/qyu/models/task.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+module Qyu
+  # A Task represents a unit of work in a workflow.
+  # Conceptually a Task:
+  # - may not exist outside the context of a queue.
+  # - it is created ON the queue
+  # - it remains on the queue until it was successfully processed (or failed "enough" times)
+  class Task
+    attr_reader :queue_name, :payload, :status, :id, :job_id, :name, :parent_task_id,
+                :message_id, :created_at, :updated_at
+
+    LEASE_PERCENTAGE_THRESHOLD_BEFORE_RENEWAL = 0.8
+    POLL_INTERVAL                             = 0.5
+
+    # @returns Task
+    # by defintion Task.create does 2 things:
+    # - persists the Task in the Store
+    # - enqueues the Task to the Queue
+    # We have to make sure that a Task is unique in the Store. Because of this
+    # create first looks up if the task has already been persisted. If it exists then
+    # there is no need to persist it again, only to enqueue it.
+    # Double (or multiple) delivery of messages is allowed and handled at worker level.
+    # Possible scenario:
+    # A Job failed at some point. A few of its tasks completed successfully, others failed.
+    # Because of this, certain tasks haven't even been created.
+    # When we restart the job, the tasks will be recreated. If a task has already existed,
+    # and completed, then that state will be unchanged, and when the worker picks it up,
+    # will notice the completed state, acknowledge the message, and continue the next steps.
+    def self.create(queue_name: nil, attributes: nil)
+      fail Qyu::Errors::InvalidTaskAttributes unless valid_attributes?(attributes)
+      fail Qyu::Errors::InvalidQueueName unless valid_queue_name?(queue_name)
+      Qyu.logger.debug "find_or_persist queue_name=#{queue_name} and attributes=#{attributes}"
+      task_id = Qyu.store.find_or_persist_task(
+        attributes['name'],
+        queue_name,
+        attributes['payload'],
+        attributes['job_id'],
+        attributes['parent_task_id']
+      ) do |t_id|
+        Qyu.logger.debug "enqueue queue_name=#{queue_name} and task_id=#{t_id}"
+        Qyu.queue.enqueue_task(queue_name, t_id)
+      end
+
+      new(task_id, attributes, queue_name)
+    end
+
+    # @returns Task
+    def self.fetch(queue_name)
+      fail Qyu::Errors::InvalidQueueName unless valid_queue_name?(queue_name)
+      begin
+        message    = Qyu.queue.fetch_next_message(queue_name)
+        task_id    = message['task_id']
+        task_attrs = Qyu.store.find_task(task_id)
+      rescue => ex
+        message ||= {}
+        raise Qyu::Errors::CouldNotFetchTask.new(queue_name, message['id'], message['task_id'], ex)
+      end
+      new(task_id, task_attrs, queue_name, message['id'])
+    end
+
+    def self.select(job_id:)
+      Qyu.store.select_tasks_by_job_id(job_id).map do |task|
+        new(task['id'], task, task['queue_name'])
+      end
+    end
+
+    def self.valid_attributes?(_attributes)
+      true
+    end
+
+    def self.valid_queue_name?(queue_name)
+      !queue_name.nil? && queue_name != ''
+    end
+
+    def acknowledgeable?
+      @status.completed? || @status.invalid_payload?
+    end
+
+    def completed?
+      @status.completed?
+    end
+
+    def locked?
+      !@lease_token.nil? && !@locked_until.nil? && Time.now < @locked_until
+    end
+
+    def lock!
+      fail Qyu::Errors::LockAlreadyAcquired if locked?
+      Qyu.logger.debug "Task with ID=#{id} lock!"
+
+      @lease_token, @locked_until = Qyu.store.lock_task!(id, Qyu.config.store[:lease_period])
+      Qyu.logger.debug "lease_token = #{@lease_token} | locked_until = #{@locked_until}"
+      return false if @lease_token.nil?
+
+      schedule_renewal
+      true
+    end
+
+    def unlock!
+      fail Qyu::Errors::LockNotAcquired unless locked?
+      Qyu.logger.debug "Task with ID=#{id} unlocking!"
+
+      @lease_thread&.kill
+      success = Qyu.store.unlock_task!(id, @lease_token)
+      if success
+        @lease_token  = nil
+        @locked_until = nil
+      end
+
+      success
+    end
+
+    def mark_queued
+      Qyu.store.update_status(id, Status::QUEUED)
+      Qyu.logger.debug "Task with ID=#{id} marked queued."
+    end
+
+    def mark_working
+      Qyu.store.update_status(id, Status::WORKING)
+      Qyu.logger.debug "Task with ID=#{id} marked working."
+    end
+
+    def mark_completed
+      Qyu.store.update_status(id, Status::COMPLETED)
+      Qyu.logger.info "Task with ID=#{id} marked completed."
+    end
+
+    def mark_failed
+      Qyu.store.update_status(id, Status::FAILED)
+      Qyu.logger.debug "Task with ID=#{id} marked failed."
+    end
+
+    def mark_invalid_payload
+      Qyu.store.update_status(id, Status::INVALID_PAYLOAD)
+      Qyu.logger.debug "Task with ID=#{id} has invalid payload."
+    end
+
+    def acknowledge_message
+      fail Qyu::Errors::MessageNotReceived if message_id.nil?
+      self.class.acknowledge_message(queue_name, message_id)
+    end
+
+    def self.acknowledge_message(queue_name, message_id)
+      Qyu.logger.debug "Acknowledging message with ID=#{message_id} from queue `#{queue_name}`"
+      Qyu.queue.acknowledge_message(queue_name, message_id)
+    end
+
+    def requeue
+      # TODO For FIFO queues (future use)
+      fail Qyu::Errors::MessageNotReceived if message_id.nil?
+      self.class.acknowledge_message(queue_name, message_id)
+      self.class.requeue(queue_name, id, message_id)
+    end
+
+    def self.requeue(queue_name, id, message_id)
+      # TODO For FIFO queues (future use)
+      Qyu.logger.debug "Re-enqueuing message with ID=#{message_id} in queue `#{queue_name}`"
+      Qyu.queue.enqueue_task(queue_name, id)
+    end
+
+    def enqueue_in_failure_queue
+      fail Qyu::Errors::MessageNotReceived if message_id.nil?
+      self.class.acknowledge_message(queue_name, message_id)
+      self.class.enqueue_in_failure_queue(queue_name, id, message_id)
+    end
+
+    def self.enqueue_in_failure_queue(queue_name, id, message_id)
+      Qyu.logger.debug "Enqueuing failed message with ID=#{message_id} in #{queue_name} failures queue"
+      Qyu.queue.enqueue_task_to_failed_queue(queue_name, id)
+    end
+
+    def job
+      @job ||= Qyu::Job.find(job_id)
+    end
+
+    def [](attribute)
+      public_send(attribute)
+    end
+
+    private
+
+    def initialize(id, attributes, queue_name, message_id = nil)
+      # puts "task initialized attrs: #{attributes}"
+      @status         = Status.new(id)
+      @id             = id
+      @job_id         = attributes['job_id']
+      @parent_task_id = attributes['parent_task_id']
+      @payload        = attributes['payload']
+      @queue_name     = queue_name
+      @message_id     = message_id
+      @name           = attributes['name']
+      @created_at     = attributes['created_at']
+      @updated_at     = attributes['updated_at']
+
+      @locked_until = nil
+      @lease_thread = nil
+      @lease_token  = nil
+    end
+
+    def schedule_renewal
+      Qyu.logger.debug 'scheduling renewal'
+      renewal_moment = Qyu::Utils.seconds_after_time(-1 * LEASE_PERCENTAGE_THRESHOLD_BEFORE_RENEWAL * Qyu.config.store[:lease_period], @locked_until)
+      Qyu.logger.debug "renewal moment: #{renewal_moment}"
+      @lease_thread = Thread.new do
+        Qyu.logger.debug 'lease thread entered'
+        while Time.now < renewal_moment
+          sleep(POLL_INTERVAL)
+          Qyu.logger.debug 'lease thread sleep'
+        end
+        Qyu.logger.debug 'lease thread time has come'
+        @locked_until = Qyu.store.renew_lock_lease(id, Qyu.config.store[:lease_period], @lease_token)
+        Qyu.logger.debug "lease thread locked until = #{@locked_until}"
+        schedule_renewal
+      end
+    end
+  end
+end

data/lib/qyu/models/workflow.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Qyu
+  class Workflow
+    attr_reader :id, :name, :descriptor, :created_at, :updated_at
+
+    class << self
+      def create(name:, descriptor:)
+        validator = Qyu::Concerns::WorkflowDescriptorValidator.new(descriptor)
+        fail Qyu::Errors::WorkflowDescriptorValidatorationError, validator.errors unless validator.valid?
+        id = persist(name, descriptor)
+        time = Time.now
+        new(id, name, descriptor, time, time)
+      end
+
+      def find(id, raise_error: true)
+        workflow_attrs = Qyu.store.find_workflow(id)
+        raise Qyu::Errors::WorkflowNotFound.new(:id, id) if workflow_attrs.nil? && raise_error
+        return nil if workflow_attrs.nil?
+        new(id, workflow_attrs['name'], workflow_attrs['descriptor'])
+      end
+
+      def find_by(name: nil, id: nil)
+        return find_by_name(name) if name
+        return find(id, raise_error: false) if id
+      end
+
+      def find_by!(name: nil, id: nil)
+        workflow = find_by(name: name, id: id)
+        raise Qyu::Errors::WorkflowNotFound.new(:id, id) if workflow.nil? && id
+        raise Qyu::Errors::WorkflowNotFound.new(:id, id) if workflow.nil? && id
+        workflow
+      end
+
+      def select(limit: 30, offset: 0, order: :asc)
+        workflow_records = Qyu.store.select_workflows(limit, offset, order)
+        workflow_records.map do |record|
+          new(record['id'], record['name'], record['descriptor'])
+        end
+      end
+
+      def delete(id)
+        Qyu.store.delete_workflow(id)
+      end
+
+      def delete_by(name: nil, id: nil)
+        raise ArgumentError, 'specify either name or id' if (name && id) || (name.nil? && id.nil?)
+        Qyu.store.delete_workflow_by_name(name) if name
+        delete(id) if id
+      end
+
+      def count
+        Qyu.store.count_workflows
+      end
+
+      private
+
+      def persist(name, descriptor)
+        Qyu.store.persist_workflow(name, descriptor)
+      end
+
+      def find_by_name(name)
+        workflow_attrs = Qyu.store.find_workflow_by_name(name)
+        return nil unless workflow_attrs
+        new(workflow_attrs['id'], workflow_attrs['name'], workflow_attrs['descriptor'])
+      end
+    end
+
+    def [](attribute)
+      public_send(attribute)
+    end
+
+    private_class_method :new
+
+    private
+
+    def initialize(id, name, descriptor, created_at = nil, updated_at = nil)
+      @id = id
+      @name = name
+      @descriptor = descriptor
+      @created_at = created_at
+      @updated_at = updated_at
+    end
+  end
+end