qyu 1.0.0

data/lib/qyu/ui/public/script.js

@@ -0,0 +1,28 @@
+(function(){
+  var collapser = function(){
+    var elem = this;
+    if ( elem.classList.contains('open') ){
+      var root = elem.dataset.root;
+      var level = elem.dataset.level;
+      document.querySelectorAll('tr[data-root="' + root + '"]').forEach(function(el){
+        var currentLevel = parseInt(el.dataset.level);
+        if (currentLevel > level) {
+          el.classList.add('collapse');
+          var parent = el.dataset.parent;
+          document.querySelector('a[data-id="' + parent + '"]').classList.remove('open');
+        }
+      });
+      elem.classList.remove('open');
+    } else {
+      var id = elem.dataset.id;
+      document.querySelectorAll('[data-parent="' + id + '"]').forEach(function(el){
+        el.classList.remove('collapse');
+      });
+      elem.classList.add('open');
+    }
+  }
+
+  document.querySelectorAll('.collapser').forEach(function(elem){
+    elem.addEventListener('click', collapser);
+  });
+}).call(this);

data/lib/qyu/ui/public/style.css

@@ -0,0 +1,6 @@
+.main-panel {
+  width: 100%;
+}
+.collapse {
+  display: none;
+}

data/lib/qyu/ui/views/footer.erb

@@ -0,0 +1,18 @@
+<footer class="footer">
+  <div class="container-fluid">
+    <nav class="pull-left">
+      <ul>
+        <li>
+          <a href="https://github.com/FindHotel/qyu">
+             Qyu is a distributed task execution system for complex workflows
+          </a>
+        </li>
+      </ul>
+    </nav>
+    <div class="copyright pull-right">
+      <a href="https://www.findhotel.net/">
+        FindHotel B.V. &copy; <%= Date.today.year %>
+      </a>
+    </div>
+  </div>
+</footer>

data/lib/qyu/ui/views/helpers/pagination.erb

@@ -0,0 +1,49 @@
+<nav aria-label="Page navigation">
+  <ul class="pagination">
+    <% if collection.page > 1 %>
+      <li>
+        <a href="<%= url("/jobs?page=1") %>">
+          &#8672; First
+        </a>
+      </li>
+      <li>
+        <a href="<%= url("/jobs?page=#{collection.page - 1}") %>" aria-label="Previous">
+          <span aria-hidden="true">&laquo;</span>
+        </a>
+      </li>
+    <% end %>
+
+    <% previous_pages_for(collection).each do |page_num| %>
+      <li>
+        <a href="<%= url("/jobs?page=#{page_num}") %>">
+          <%= page_num %>
+        </a>
+      </li>
+    <% end %>
+    <li class="active">
+      <a href="#">
+        <%= collection.page %> <span class="sr-only">(current)</span>
+      </a>
+    </li>
+    <% next_pages_for(collection).each do |page_num| %>
+      <li>
+        <a href="<%= url("/jobs?page=#{page_num}") %>">
+          <%= page_num %>
+        </a>
+      </li>
+    <% end %>
+
+    <% if collection.total_pages > collection.page %>
+      <li>
+        <a href="<%= url("/jobs?page=#{collection.page + 1}") %>" aria-label="Next">
+          <span aria-hidden="true">&raquo;</span>
+        </a>
+      </li>
+      <li>
+        <a href="<%= url("/jobs?page=#{collection.total_pages}" ) %>">
+          Last &#8674;
+        </a>
+      </li>
+    <% end %>
+  </ul>
+</nav>

data/lib/qyu/ui/views/jobs.erb

@@ -0,0 +1,58 @@
+<div class="row">
+  <div class="col-md-12">
+    <div class="card">
+      <div class="header">
+        <h4 class="title">Jobs</h4>
+        <!-- <p class="category">Last 24 Hours</p> -->
+      </div>
+      <div class="content">
+        <table class="table table-bordered">
+          <thead>
+            <tr>
+              <th>#</th>
+              <th>Description</th>
+              <th>Payload</th>
+            </tr>
+          </thead>
+          <tbody>
+            <% jobs.each do |job| %>
+              <tr>
+                <td>
+                  <a href="<%= url("/jobs/#{job.id}") %>">
+                    <p><%= job.id %></p>
+                    <p>Created: <%= job.created_at %></p>
+                  </a>
+                </td>
+                <td>
+                  <strong>Starts:</strong>
+                  <ul>
+                    <% job.descriptor['starts'].each do |entry_point| %>
+                      <li><%= entry_point %></li>
+                    <% end %>
+                  </ul>
+                  <strong>Tasks:</strong>
+                  <ul>
+                    <% job.descriptor['tasks'].keys.each do |task| %>
+                      <li><%= task %></li>
+                    <% end %>
+                  </ul>
+                </td>
+                <td>
+                  <ul>
+                    <% job.payload.each do |key, value| %>
+                      <li><%= key %>: <%= value %></li>
+                    <% end %>
+                  </ul>
+                </td>
+              </tr>
+            <% end %>
+          </tbody>
+        </table>
+
+        <nav aria-label="Page navigation" class="text-center">
+          <%= erb :'helpers/pagination', locals: { collection: jobs } %>
+        </nav>
+      </div>
+    </div>
+  </div>
+</div>

data/lib/qyu/ui/views/kaminari/_first_page.html.erb

@@ -0,0 +1,3 @@
+<li>
+  <%= link_to_unless current_page.first?, raw(t 'views.pagination.first'), url, :remote => remote %>
+</li>

data/lib/qyu/ui/views/kaminari/_gap.html.erb

@@ -0,0 +1,3 @@
+<li class='disabled'>
+  <%= content_tag :a, raw(t 'views.pagination.truncate') %>
+</li>

data/lib/qyu/ui/views/kaminari/_last_page.html.erb

@@ -0,0 +1,3 @@
+<li>
+  <%= link_to_unless current_page.last?, raw(t 'views.pagination.last'), url, {:remote => remote} %>
+</li>

data/lib/qyu/ui/views/kaminari/_next_page.html.erb

@@ -0,0 +1,3 @@
+<li>
+  <%= link_to_unless current_page.last?, raw(t 'views.pagination.next'), url, :rel => 'next', :remote => remote %>
+</li>

data/lib/qyu/ui/views/kaminari/_page.html.erb

@@ -0,0 +1,9 @@
+<% if page.current? %>
+  <li class='active'>
+    <%= content_tag :a, page, remote: remote, rel: (page.next? ? 'next' : (page.prev? ? 'prev' : nil)) %>
+  </li>
+<% else %>
+  <li>
+    <%= link_to page, url, remote: remote, rel: (page.next? ? 'next' : (page.prev? ? 'prev' : nil)) %>
+  </li>
+<% end %>

data/lib/qyu/ui/views/kaminari/_paginator.html.erb

@@ -0,0 +1,15 @@
+<%= paginator.render do -%>
+  <ul class="pagination">
+    <%= first_page_tag unless current_page.first? %>
+    <%= prev_page_tag unless current_page.first? %>
+    <% each_page do |page| -%>
+      <% if page.left_outer? || page.right_outer? || page.inside_window? -%>
+        <%= page_tag page %>
+      <% elsif !page.was_truncated? -%>
+        <%= gap_tag %>
+      <% end -%>
+    <% end -%>
+    <%= next_page_tag unless current_page.last? %>
+    <%= last_page_tag unless current_page.last? %>
+  </ul>
+<% end -%>

data/lib/qyu/ui/views/kaminari/_prev_page.html.erb

@@ -0,0 +1,3 @@
+<li>
+  <%= link_to_unless current_page.first?, raw(t 'views.pagination.previous'), url, :rel => 'prev', :remote => remote %>
+</li>

data/lib/qyu/ui/views/layout.erb

@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
+
+    <title>Qyu Dashboard</title>
+
+    <meta content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0' name='viewport' />
+    <meta name="viewport" content="width=device-width" />
+
+    <!-- Bootstrap core CSS     -->
+    <link rel="stylesheet" type="text/css" href="<%= url("/bootstrap.min.css")%>">
+    <!--  Paper Dashboard core CSS    -->
+    <link rel="stylesheet" type="text/css" href="<%= url("/paper-dashboard.css")%>">
+    <!--  custom CSS    -->
+    <link rel="stylesheet" type="text/css" href="<%= url("/style.css")%>">
+  </head>
+  <body>
+    <div class="wrapper">
+      <div class="main-panel">
+        <%= erb :navbar %>
+        <div class="content">
+          <div class="container-fluid">
+            <%= yield %>
+          </div>
+        </div>
+        <%= erb :footer %>
+      </div>
+    </div>
+    <script src="<%= url("/script.js")%>"></script>
+  </body>
+</html>

data/lib/qyu/ui/views/navbar.erb

@@ -0,0 +1,29 @@
+<nav class="navbar navbar-default">
+  <div class="container-fluid">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar bar1"></span>
+        <span class="icon-bar bar2"></span>
+        <span class="icon-bar bar3"></span>
+      </button>
+      <a class="navbar-brand" href="/">Qyu</a>
+    </div>
+    <div class="collapse navbar-collapse">
+      <ul class="nav navbar-nav navbar-right">
+        <li class="active">
+          <a href="#">
+            <i class="ti-panel"></i>
+            <p>Jobs</p>
+          </a>
+        </li>
+        <li>
+          <a href="#">
+            <i class="ti-settings"></i>
+            <p>Tasks</p>
+          </a>
+        </li>
+      </ul>
+    </div>
+  </div>
+</nav>

data/lib/qyu/ui/views/pagination.erb

@@ -0,0 +1,19 @@
+<nav aria-label="Page navigation" class="text-center">
+  <ul class="pagination">
+    <li>
+      <a href="#" aria-label="Previous">
+        <span aria-hidden="true">«</span>
+      </a>
+    </li>
+    <li><a href="#">1</a></li>
+    <li><a href="#">2</a></li>
+    <li><a href="#">3</a></li>
+    <li><a href="#">4</a></li>
+    <li><a href="#">5</a></li>
+    <li>
+      <a href="#" aria-label="Next">
+        <span aria-hidden="true">»</span>
+      </a>
+    </li>
+  </ul>
+</nav>

data/lib/qyu/ui/views/show_job.erb

@@ -0,0 +1,55 @@
+<div class="row">
+  <div class="col-md-12">
+    <div class="card">
+      <div class="header">
+        <h4 class="title">Job #<%= job.id %></h4>
+        <p class="category">
+          Tasks list: <%= total_count %> <%= 'task'.pluralize(total_count) %>
+        </p>
+      </div>
+      <div class="content">
+        <table class="table table-stripped">
+          <thead>
+            <tr>
+              <th>Task Name</th>
+              <th>Queued</th>
+              <th>Working</th>
+              <th>Completed</th>
+              <th>Failed</th>
+              <th>Invalid Payload</th>
+            </tr>
+          </thead>
+          <tbody>
+            <% task_statuses.each do |task_name, statuses| %>
+              <tr>
+                <td><%= task_name %></td>
+                <td><%= statuses['queued'] %></td>
+                <td><%= statuses['working'] %></td>
+                <td><%= statuses['completed'] %></td>
+                <td><%= statuses['failed'] %></td>
+                <td><%= statuses['invalid_payload'] %></td>
+              </tr>
+            <% end %>
+          </tbody>
+        </table>
+        <table class="table table-stripped">
+          <thead>
+            <tr>
+              <th>Name</th>
+              <th>Queue Name</th>
+              <th>Status</th>
+              <th>Payload</th>
+              <!-- <th>Locked Until</th> -->
+              <!-- <th>Locked By</th> -->
+            </tr>
+          </thead>
+          <tbody>
+            <% (tasks[nil] || []).each do |task| %>
+              <%= erb :task_row, locals: { tasks: tasks, task: task, level: 0, root_id: task.id } %>
+            <% end %>
+          </tbody>
+        </table>
+      </div>
+    </div>
+  </div>
+</div>

data/lib/qyu/ui/views/sidebar.erb

@@ -0,0 +1,17 @@
+<div class="sidebar" data-background-color="white" data-active-color="danger">
+  <div class="sidebar-wrapper">
+    <div class="logo">
+      <a href="/" class="simple-text">
+        Arc Yu
+      </a>
+    </div>
+
+    <ul class="nav">
+      <li class="active">
+        <a href="/">
+          <p>Jobs</p>
+        </a>
+      </li>
+    </ul>
+  </div>
+</div>

data/lib/qyu/ui/views/task_row.erb

@@ -0,0 +1,26 @@
+<tr data-parent="<%= task.parent_task_id %>"
+    data-level="<%= level %>"
+    data-root="<%= root_id %>"
+    class="<%= 'collapse' if level > 0 %>">
+  <td>
+    <%= raw_html("•&nbsp;&nbsp;" * level) if level > 0  %>
+    <% if tasks[task.id] %>
+      <a href="#" class="collapser"
+         data-id="<%= task.id %>"
+         data-level="<%= level %>"
+         data-root="<%= root_id %>">
+        <%= task.name %>
+      </a>
+    <% else %>
+      <%= task.name %>
+    <% end %>
+  </td>
+  <td><%= task.queue_name %></td>
+  <td><%= task.status.status %></td>
+  <td><%= task.payload %></td>
+  <!-- <td><%# task.locked_until %></td> -->
+  <!-- <td><%# task.locked_by %></td> -->
+</tr>
+<% (tasks[task.id] || []).each do |t| %>
+  <%= erb :task_row, locals: { tasks: tasks, task: t, level: level + 1, root_id: root_id } %>
+<% end %>

data/lib/qyu/utils.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Qyu
+  module Utils
+    def self.seconds_after_time(seconds, start_time = Time.now)
+      start_time + seconds
+    end
+
+    def self.uuid
+      SecureRandom.uuid
+    end
+
+    def self.stringify_hash_keys(object)
+      object.map { |k, v| [k.to_s, v] }.to_h
+    end
+  end
+end

data/lib/qyu/version.rb

@@ -0,0 +1,3 @@
+module Qyu
+  VERSION = '1.0.0'
+end

data/lib/qyu/workers.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'qyu/workers/concerns/callback'
+require 'qyu/workers/concerns/failure_queue'
+require 'qyu/workers/concerns/payload_validator'
+require 'qyu/workers/base'
+require 'qyu/workers/sync'
+
+Qyu::Worker = Qyu::Workers::Base
+Qyu::SyncWorker = Qyu::Workers::Sync

data/lib/qyu/workers/base.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Qyu
+  module Workers
+    class Base
+      include Concerns::Callback
+      include Concerns::PayloadValidator
+      include Concerns::FailureQueue
+
+      attr_reader :id
+      attr_accessor :processed_tasks
+
+      def initialize(&block)
+        @id = Qyu::Utils.uuid
+        @processed_tasks = 0
+        instance_exec(&block) if block_given?
+      end
+
+      def work(queue_name, blocking: true)
+        log(:info, "Worker started for queue '#{queue_name}'")
+        repeat = true
+
+        remaining_fetch_retries = 3
+
+        while repeat
+          run_callbacks(:execute) do
+            begin
+              fetched_task = fetch_task(queue_name)
+              validate_payload!(fetched_task)
+              log(:info, "Worker processed #{processed_tasks} tasks from queue `#{queue_name}`")
+              if fetched_task.acknowledgeable?
+                discard_completed_task(fetched_task)
+              elsif fetched_task.lock!
+                fetched_task.mark_working
+                begin
+                  yield(fetched_task)
+                  conclude_task(fetched_task)
+                rescue => ex
+                  fail_task(fetched_task, ex)
+                end
+              end
+            rescue Qyu::Errors::UnsyncError
+            rescue Qyu::Errors::CouldNotFetchTask => ex
+              if remaining_fetch_retries <= 0
+                acknowledge_message_with_task_id_not_found_in_store(ex)
+              else
+                sleep(remaining_fetch_retries)
+                remaining_fetch_retries -= 1
+                retry
+              end
+            rescue Qyu::Errors::PayloadValidationError
+              fetched_task.mark_invalid_payload
+            rescue => ex
+              log("Worker error: #{ex.class}: #{ex.message}")
+              log("Backtrace: #{ex.backtrace.join("\n")}")
+            end
+          end
+
+          repeat = blocking
+          run_garbage_collector
+        end
+      end
+
+      private
+
+      def fetch_task(queue_name)
+        fetched_task = Qyu::Task.fetch(queue_name)
+        @processed_tasks += 1
+        fetched_task
+      end
+
+      def discard_completed_task(fetched_task)
+        log(:debug, 'Fetched completed task. Discarding...')
+        fetched_task.acknowledge_message
+      end
+
+      def conclude_task(fetched_task)
+        Qyu.store.transaction do
+          log(:debug, 'Task finished. Creating next tasks.')
+          fetched_task.job.create_next_tasks(
+            fetched_task,
+            fetched_task.job.payload.merge(fetched_task.payload)
+          )
+          fetched_task.unlock!
+          fetched_task.mark_completed
+        end
+        fetched_task.acknowledge_message
+      end
+
+      def fail_task(fetched_task, exception)
+        unless exception.class == Qyu::Errors::UnsyncError
+          log("Worker error: #{exception.class}: #{exception.message}")
+          log("Backtrace: #{exception.backtrace.join("\n")}")
+        end
+        Qyu.store.transaction do
+          fetched_task.enqueue_in_failure_queue if @failure_queue
+          fetched_task.unlock!
+          fetched_task.mark_queued
+        end
+      end
+
+      def acknowledge_message_with_task_id_not_found_in_store(exception)
+        # If a task is not found in the Store then there is no point attempting
+        # to fetch the message over and over again.
+        log("Worker error: #{exception.class}: #{exception.message}")
+        log("Backtrace: #{exception.backtrace.join("\n")}")
+        log("Original error: #{exception.original_error.class}: #{exception.original_error.message}")
+        log("Backtrace: #{exception.original_error.backtrace.join("\n")}")
+        if exception.original_error.class == Qyu::Errors::TaskNotFound &&
+           exception.queue_name &&
+           exception.message_id
+          Qyu::Task.acknowledge_message(exception.queue_name, exception.message_id)
+        end
+      end
+
+      def log(level = :error, message)
+        Qyu.logger.public_send(level, "[#{id}] #{message}")
+      end
+
+      def run_garbage_collector
+        log(:debug, 'Running garbage collector')
+        GC.start
+      end
+    end
+  end
+end