qyu 1.0.0

data/lib/qyu/queue.rb

@@ -0,0 +1,5 @@
+require 'qyu/queue/base'
+require 'qyu/queue/memory/adapter'
+
+Qyu::Config::QueueConfig.register(Qyu::Queue::Memory::Adapter)
+Qyu::Factory::QueueFactory.register(Qyu::Queue::Memory::Adapter)

data/lib/qyu/queue/base.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Qyu
+  module Queue
+    # Qyu::Queue::Base
+    class Base
+      # This class acts as an interface for any queue adapter implemented for Qyu
+      # Implement the following methods in any queue and it should work seemlessly
+      def self.valid_config?(_config)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      # Instance methods
+      def enqueue_tasks(queue_name, task_ids)
+        task_ids.each do |task_id|
+          enqueue_task(queue_name, task_id)
+        end
+      end
+
+      # Instance methods to override
+      def enqueue_task(_queue_name, _task_id)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def enqueue_task_to_failed_queue(_queue_name, _task_id)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def fetch_next_message(_queue_name)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def acknowledge_message(_queue_name, _message_id)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def queues
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def size(_queue_name)
+        fail Qyu::Errors::NotImplementedError
+      end
+    end
+  end
+end

data/lib/qyu/queue/memory/adapter.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Qyu
+  module Queue
+    module Memory
+      class Adapter < Qyu::Queue::Base
+        TYPE = :memory
+
+        def initialize(_config)
+          @temp_store = Hash.new(false)
+          @queues = {}
+          @threads = []
+        end
+
+        def self.valid_config?(_config)
+          # TODO
+          true
+        end
+
+        def enqueue_task(queue_name, task_id)
+          queue(queue_name) << { 'task_id' => task_id }
+        end
+
+        def enqueue_task_to_failed_queue(queue_name, task_id)
+          failed_queue_name = queue_name + '-failed'
+          enqueue_task(failed_queue_name, task_id)
+        end
+
+        # fetch_next_message
+        #
+        # @param [String] queue_name
+        # @return [Hash] the acknowledge message
+        #
+        # TODO Note the uglyness in `while ... empty?`; it's because of reasons
+        # mainly for this (http://stackoverflow.com/q/11660253) reason.
+        def fetch_next_message(queue_name)
+          sleep(1) while queue(queue_name).empty?
+          message = queue(queue_name).pop(true)
+          message_id = Qyu::Utils.uuid
+          schedule_requeue(message, message_id, queue_name)
+          {
+            'id' => message_id,
+            'task_id' => message['task_id']
+          }
+        end
+
+        def acknowledge_message(_queue_name, message_id)
+          @temp_store[message_id] = true
+        end
+
+        def queues
+          @queues.map do |name, queue|
+            { name: name, messages: queue&.size }
+          end
+        end
+
+        def size(queue_name)
+          queue(queue_name).size
+        end
+
+        private
+
+        def schedule_requeue(message, message_id, queue_name)
+          @threads << Thread.new(message_id, message) do |t_message_id, t_message|
+            sleep(5)
+            queue(queue_name) << t_message unless message_acknowledged?(t_message_id)
+          end
+        end
+
+        def message_acknowledged?(message_id)
+          @temp_store[message_id] == true
+        end
+
+        # queue, or "get_or_create_queue"
+        #
+        # @param [String] name The name of the queue to create if it does
+        #    does not exist and return;
+        def queue(name)
+          if @queues[name]
+            Qyu.logger.debug "Queue `#{name}`: #{@queues[name].length} elements"
+            return @queues[name]
+          end
+          Qyu.logger.info "Could not find queue `#{name}`, creating it"
+          @queues[name] ||= ::Queue.new
+        end
+        alias get_or_create_queue queue
+      end
+    end
+  end
+end

data/lib/qyu/store.rb

@@ -0,0 +1,5 @@
+require 'qyu/store/base'
+require 'qyu/store/memory/adapter'
+
+Qyu::Config::StoreConfig.register(Qyu::Store::Memory::Adapter)
+Qyu::Factory::StoreFactory.register(Qyu::Store::Memory::Adapter)

data/lib/qyu/store/base.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Qyu
+  module Store
+    # Qyu::Store::Base
+    class Base
+      # This class acts as an interface for any store implemented for Qyu
+      # Implement the following methods in any store and it should work seemlessly
+      def self.valid_config?(_config)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def transaction
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      ## Workflow
+      def persist_workflow(name, descriptor)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def find_workflow(_id)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def find_workflow_by_name(_name)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def delete_workflow(_id)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def delete_workflow_by_name(name)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      ## Job
+      def persist_job(_workflow, _payload)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def find_job(_id)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def select_jobs(_limit, _offset, _order = :asc)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def count_jobs
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def delete_job(_id)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def clear_completed_jobs
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      ## Task
+
+      def find_or_persist_task(_name, _payload, _job_id, _parent_task_id)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def find_task(_id)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def find_task_ids_by_job_id_and_name(_job_id, _name)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def find_task_ids_by_job_id_name_and_parent_task_ids(_job_id, _name, _parent_task_ids)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def lock_task!(_id, _lease_time)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def unlock_task!(_id, _lease_token)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def renew_lock_lease(_id, _lease_time, _lease_token)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def update_status(_id, _status)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def task_status_counts(_job_id)
+        fail Qyu::Errors::NotImplementedError
+      end
+
+      def select_tasks_by_job_id
+        fail Qyu::Errors::NotImplementedError
+      end
+    end
+  end
+end

data/lib/qyu/store/memory/adapter.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+module Qyu
+  module Store
+    module Memory
+      class Adapter < Qyu::Store::Base
+        TYPE = :memory
+
+        def initialize(_config)
+          @workflows = {}
+          @jobs = {}
+          @tasks = {}
+          @locks = {}
+          @semaphore = Mutex.new
+        end
+
+        def self.valid_config?(_config)
+          # TODO
+          true
+        end
+
+        def find_workflow(id)
+          @workflows[id]
+        end
+
+        def find_workflow_by_name(name)
+          @workflows.detect do |_id, wflow|
+            wflow['name'] == name
+          end.last
+        end
+
+        def persist_workflow(name, descriptor)
+          id = Qyu::Utils.uuid
+          @workflows[id] = {
+            'id'         => id,
+            'name'       => name,
+            'descriptor' => descriptor
+          }
+          id
+        end
+
+        def delete_workflow(id)
+          @workflows.delete(id)
+        end
+
+        def delete_workflow_by_name(name)
+          workflow = find_workflow_by_name(name)
+          return unless workflow
+          delete_workflow(workflow['id'])
+        end
+
+        def find_job(id)
+          @jobs[id]
+        end
+
+        def select_jobs(limit, offset, order = :asc)
+          ids = @jobs.keys[offset, limit]
+          selected = ids.map { |id| { id: id }.merge(@jobs[id]) }
+          return selected if order == :asc
+          selected.reverse
+        end
+
+        def persist_job(workflow, payload)
+          id = Qyu::Utils.uuid
+          @jobs[id] = {
+            'payload'  => payload,
+            'workflow' => workflow
+          }
+          id
+        end
+
+        def delete_job(id)
+          @jobs.delete(id)
+        end
+
+        def clear_completed_jobs
+          # TODO
+        end
+
+        def count_jobs
+          @jobs.count
+        end
+
+        ## Task methods
+        def find_task(id)
+          @tasks[id]
+        end
+
+        def find_or_persist_task(name, queue_name, payload, job_id, parent_task_id)
+          matching_task = @tasks.detect do |_id, attrs|
+            attrs['job_id'] == job_id \
+            && attrs['name'] == name \
+            && attrs['payload'] == payload \
+            && attrs['queue_name'] == queue_name \
+            && attrs['parent_task_id'] == parent_task_id
+          end
+          return matching_task[0] if matching_task
+
+          id = Qyu::Utils.uuid
+          @tasks[id] = {
+            'name' => name,
+            'queue_name' => queue_name,
+            'parent_task_id' => parent_task_id,
+            'status' => Qyu::Status::QUEUED,
+            'payload' => payload,
+            'job_id' => job_id
+          }
+          yield(id)
+          id
+        end
+
+        def find_task_ids_by_job_id_and_name(job_id, name)
+          @tasks.select do |_id, attrs|
+            attrs['job_id'] == job_id && attrs['name'] == name
+          end.map { |(id, _attr)| id }
+        end
+
+        def find_task_ids_by_job_id_name_and_parent_task_ids(job_id, name, parent_task_ids)
+          @tasks.select do |_id, attrs|
+            attrs['job_id'] == job_id &&
+            attrs['name'] == name &&
+            parent_task_ids.include?(attrs['parent_task_id'])
+          end.map { |(id, _attr)| id }
+        end
+
+        def select_tasks_by_job_id(job_id)
+          @tasks.select { |_id, attrs| attrs['job_id'] == job_id }.map { |id, attrs| attrs.merge('id' => id) }
+        end
+
+        def task_status_counts(job_id)
+          # TODO
+          {}
+        end
+
+        def lock_task!(id, lease_time)
+          uuid = Qyu::Utils.uuid
+          locked = false
+          locked_until = nil
+          @semaphore.synchronize do
+            if @locks[id].nil? || @locks[id][:locked_until] < Time.now
+              locked_until = Qyu::Utils.seconds_after_time(lease_time)
+              @locks[id] = { locked_by: uuid, locked_until: locked_until }
+              locked = true
+            end
+          end
+
+          return [nil, nil] unless locked
+
+          [uuid, locked_until]
+        end
+
+        def unlock_task!(id, lease_token)
+          unlocked = false
+          @semaphore.synchronize do
+            if @locks[id][:locked_by] == lease_token
+              @locks.delete(id)
+              unlocked = true
+            end
+          end
+
+          unlocked
+        end
+
+        def update_status(id, status)
+          @tasks[id]['status'] = status
+        end
+
+        def renew_lock_lease(id, lease_time, lease_token)
+          locked_until = nil
+          @semaphore.synchronize do
+            if @locks[id][:locked_by] == lease_token && Time.now <= @locks[id][:locked_until]
+              locked_until = Qyu::Utils.seconds_after_time(lease_time)
+              @locks[id] = { locked_by: lease_token, locked_until: locked_until }
+            end
+          end
+
+          locked_until
+        end
+
+        def transaction
+          # TODO
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/qyu/ui.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'sinatra'
+require 'qyu/ui/helpers/pagination'
+
+module Qyu
+  class UI < Sinatra::Base
+    set :port, ENV['PORT'] || 3000
+    set :host, ENV['HOST'] || '0.0.0.0'
+
+    set :views, "#{__dir__}/ui/views"
+    set :public_folder, "#{__dir__}/ui/public"
+
+    include Qyu::Helpers::Pagination
+
+    get '/' do
+      redirect to('/jobs')
+    end
+
+    get '/jobs' do
+      page = params[:page].to_i > 0 ? params[:page].to_i : 1
+      limit = 10
+      offset = (page - 1) * limit
+
+      jobs = PaginatableArray.new(
+        Qyu::Job.select(limit: limit, offset: offset, order: :desc),
+        limit: limit, offset: offset, total_count: Qyu::Job.count,
+        page: page
+      )
+
+      erb :jobs, layout: true, locals: { jobs: jobs }
+    end
+
+    get '/jobs/:id' do
+      job = Qyu::Job.find(params[:id])
+      tasks_records = Qyu::Task.select(job_id: job.id)
+      total_count = tasks_records.count
+      task_statuses = job.task_status_counts
+      tasks = tasks_records.group_by(&:parent_task_id)
+      erb :show_job,
+        layout: true,
+        locals: {
+          job: job,
+          tasks: tasks,
+          total_count: total_count,
+          task_statuses: task_statuses
+        }
+    end
+
+    private
+
+    def raw_html(value)
+      String.respond_to?(:html_safe) ? value.html_safe : value
+    end
+  end
+end