joblin 0.1.0

data/lib/joblin/batching/compat/sidekiq/web/views/_batches_table.erb

@@ -0,0 +1,44 @@
+<table class="table table-striped table-bordered table-hover">
+    <thead>
+        <tr>
+            <th rowspan="2"><%= t('Started') %></th>
+            <th rowspan="2"><%= t('BID') %></th>
+            <th rowspan="2"><%= t('Description') %></th>
+
+            <th colspan="5"><%= t('Jobs') %></th>
+            <th colspan="4"><%= t('Sub-Batches') %></th>
+        </tr>
+        <tr>
+            <th><%= t('Pending') %></th>
+            <th><%= t('Failed') %></th>
+            <th><%= t('Dead') %></th>
+            <th><%= t('Complete') %></th>
+            <th><%= t('Total') %></th>
+
+            <th><%= t('Pending') %></th>
+            <th><%= t('Failed') %></th>
+            <th><%= t('Success') %></th>
+            <th><%= t('Total') %></th>
+        </tr>
+    </thead>
+
+    <% batches.each do |batch| %>
+        <% status = Joblin::Batching::Batch::Status.new(batch) %>
+        <tr>
+            <td><%= safe_relative_time(batch.created_at.to_f) %></th>
+            <td><a href="<%= root_path %>batches/<%= batch.bid %>"><%= batch.bid %></a></td>
+            <td><%= batch.description %></th>
+
+            <td><%= status.pending %></th>
+            <td><%= status.failures %></th>
+            <td><%= status.dead %></th>
+            <td><%= status.completed_count %></th>
+            <td><%= status.job_count %></th>
+
+            <td><%= status.child_count - status.successful_children_count %></th>
+            <td><%= status.failed_children_count %></th>
+            <td><%= status.successful_children_count %></th>
+            <td><%= status.child_count %></th>
+        </tr>
+    <% end %>
+</table>

data/lib/joblin/batching/compat/sidekiq/web/views/_common.erb

@@ -0,0 +1,13 @@
+
+<% add_to_head do %>
+    <meta name="sidekiq-baseurl" content="<%= root_path %>" />
+    <% if dev_mode? %>
+        <link href="<%= root_path %>batches_assets/css/styles.less" media="screen" rel="stylesheet/less" type="text/css" />
+        <script async type="text/javascript" src="https://unpkg.com/less@4.1.1/dist/less.js" data-async="true"></script>
+    <% else %>
+        <link href="<%= root_path %>batches_assets/css/styles.less" media="screen" rel="stylesheet/less" type="text/css" />
+        <script async type="text/javascript" src="https://unpkg.com/less@4.1.1/dist/less.js" data-async="true"></script>
+        <%# TODO: Pre-compile LESS %>
+        <!-- <link href="<%= root_path %>batches_assets/css/styles.css" media="screen" rel="stylesheet" type="text/css" /> -->
+    <% end %>
+<% end %>

data/lib/joblin/batching/compat/sidekiq/web/views/_jobs_table.erb

@@ -0,0 +1,21 @@
+<table class="table table-striped table-bordered table-hover">
+    <thead>
+        <tr>
+            <th><%= t('JID') %></th>
+            <th><%= t('Job Class') %></th>
+            <th><%= t('Parameters') %></th>
+        </tr>
+    </thead>
+
+    <% @jobs.each do |job_desc| %>
+        <tr>
+            <td><%= job_desc[:jid] %></td>
+            <td><%= job_desc['job'] %></td>
+            <td>
+                <code class="code-wrap">
+                    <div class="args-extended"><%= job_desc['parameters'].to_json  %></div>
+                </code>
+            </td>
+        </tr>
+    <% end %>
+</table>

data/lib/joblin/batching/compat/sidekiq/web/views/_pagination.erb

@@ -0,0 +1,26 @@
+
+<% if @total_size > @count %>
+  <ul class="pagination pull-right flip">
+    <% _page_key = defined?(page_key) ? page_key.to_s : 'page' %>
+
+    <li class="<%= 'disabled' if @current_page == 1 %>">
+      <a href="<%= url %>?<%= _page_key %>=1">&laquo;</a>
+    </li>
+    <% if @current_page > 1 %>
+      <li>
+        <a href="<%= url %>?<%= (qparams(page: @current_page - 1) || '').gsub('page', _page_key) %>"><%= @current_page - 1 %></a>
+      </li>
+    <% end %>
+    <li class="disabled">
+      <a href="<%= url %>?<%= (qparams(page: @current_page) || '').gsub('page', _page_key) %>"><%= @current_page %></a>
+    </li>
+    <% if @total_size > @current_page * @count %>
+      <li>
+        <a href="<%= url %>?<%= (qparams(page: @current_page + 1) || '').gsub('page', _page_key) %>"><%= @current_page + 1 %></a>
+      </li>
+    <% end %>
+    <li class="<%= 'disabled' if @total_size <= @current_page * @count %>">
+      <a href="<%= url %>?<%= (qparams(page: (@total_size.to_f / @count).ceil) || '').gsub('page', _page_key) %>">&raquo;</a>
+    </li>
+  </ul>
+<% end %>

data/lib/joblin/batching/compat/sidekiq/web/views/batch.erb

@@ -0,0 +1,81 @@
+<%= erb get_template(:_common) %>
+
+<h3><%= t('Batch') %></h3>
+<% status = Joblin::Batching::Batch::Status.new(@batch) %>
+
+<div class="table_container">
+    <table class="table table-striped table-bordered table-white table-hover">
+        <tbody>
+            <tr>
+                <th colspan="2" scope=row><%= t('Batch') %></td>
+                <td><%= @batch.bid %></td>
+            </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Parent') %></td>
+                <td>
+                    <% if @batch.parent_bid.present? %>
+                        <a href="<%= root_path %>batches/<%= @batch.parent_bid %>"><%= @batch.parent_bid %></a>
+                    <% else %>
+                        ROOT
+                    <% end %>
+                </td>
+            </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Started') %></td>
+                <td><%= safe_relative_time(@batch.created_at.to_f) %></td>
+            </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Description') %></td>
+                <td><%= @batch.description %></td>
+            </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Context') %></td>
+                <td>
+                    <code class="code-wrap batch-context">
+                        <div class="args-extended"><%= format_context(@batch) %></div>
+                    </code>
+                </td>
+            </tr>
+        </tbody>
+    </table>
+</div>
+
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Child Batches') %>
+        </h3>
+    </div>
+</header>
+
+<%= erb get_template(:_batch_tree), locals: { } %>
+
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Jobs') %>
+            <i>(Queued and Current)</i>
+        </h3>
+    </div>
+    <%
+        @current_page = @current_jobs_page
+        @total_size = @total_jobs_size
+    %>
+    <% if @jobs.any? && @total_size > @count.to_i %>
+        <div class="col-sm-4">
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches/#{@batch.bid}", page_key: :job_page } %>
+        </div>
+    <% end %>
+</header>
+
+<% if @jobs.any? %>
+<div class="table_container">
+    <%= erb get_template(:_jobs_table), locals: { jobs: @jobs } %>
+</div>
+<% end %>
+
+<form class="form-horizontal" action="<%= root_path %>batches/<%= @batch.bid %>" method="post">
+    <%= csrf_tag %>
+    <a class="btn btn-default" href="<%= root_path %>batches"><%= t('GoBack') %></a>
+    <input class="btn btn-danger" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>

data/lib/joblin/batching/compat/sidekiq/web/views/batches.erb

@@ -0,0 +1,23 @@
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Batches') %>
+        </h3>
+    </div>
+    <% if @batches.any? && @total_size > @count.to_i %>
+        <div class="col-sm-4">
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}batches" } %>
+        </div>
+    <% end %>
+</header>
+
+<% if @batches.any? %>
+<div class="table_container">
+    <%= erb get_template(:_batches_table), locals: { batches: @batches } %>
+</div>
+<% end %>
+
+<form action="<%= root_path %>batches/all" method="post">
+    <%= csrf_tag %>
+    <input class="btn btn-danger btn-xs pull-right flip" type="submit" name="delete" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>

data/lib/joblin/batching/compat/sidekiq/web/views/pool.erb

@@ -0,0 +1,137 @@
+<h3><%= t('Pool') %></h3>
+
+<div class="table_container">
+    <table class="table table-striped table-bordered table-white table-hover">
+        <tbody>
+            <tr>
+                <th scope=row><%= t('PID') %></td>
+                <td><%= @pool.pid %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Created') %></td>
+                <td><%= safe_relative_time(@pool.created_at.to_f) %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Description') %></td>
+                <td><%= @pool.description %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Type') %></td>
+                <td><%= @pool.order.to_s.upcase %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Concurrency') %></td>
+                <td><%= @pool.concurrency %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Utilization') %></td>
+                <td><%= @pool.active_count %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Pending Tasks') %></td>
+                <td><%= @pool.pending_count %></td>
+            </tr>
+            <tr>
+                <th scope=row><%= t('Returned Tasks') %></td>
+                <td><%= @pool.complete_count %></td>
+            </tr>
+        </tbody>
+    </table>
+</div>
+
+<%# Queued/Active Tasks %>
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Queued/Active Tasks') %>
+        </h3>
+    </div>
+</header>
+
+<% if @active_tasks.any? %>
+<div class="table_container">
+    <table class="table table-striped table-bordered table-hover">
+        <thead>
+            <tr>
+                <th><%= t('Wrapper Batch BID') %></th>
+                <th><%= t('Job Class') %></th>
+                <th><%= t('Parameters') %></th>
+            </tr>
+        </thead>
+
+        <% @active_tasks.each do |job_desc| %>
+            <tr>
+                <td><a href="<%= root_path %>batches/<%= job_desc['pool_wrapper_batch'] %>"><%= job_desc['pool_wrapper_batch'] %></a></td>
+                <td><%= job_desc['job'] %></td>
+                <td>
+                    <code class="code-wrap">
+                        <div class="args-extended">
+                            <%= job_desc['args']&.to_json %>
+                            <%= job_desc['kwargs']&.to_json %>
+                        </div>
+                    </code>
+                </td>
+            </tr>
+        <% end %>
+    </table>
+</div>
+<% end %>
+
+<%# Pending Tasks %>
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Pending Tasks') %>
+        </h3>
+    </div>
+    <%
+        @current_page = @current_jobs_page
+        @total_size = @total_jobs_size
+    %>
+    <% if @jobs.any? && @total_size > @count.to_i %>
+        <div class="col-sm-4">
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}pools/#{@pool.pid}", page_key: :job_page } %>
+        </div>
+    <% end %>
+</header>
+
+<% if @jobs.any? %>
+<div class="table_container">
+    <table class="table table-striped table-bordered table-hover">
+        <thead>
+            <tr>
+                <th><%= t('Job Class') %></th>
+                <th><%= t('Parameters') %></th>
+                <th><%= t('Wrapper Batch BID') %></th>
+                <% if @pool.order == 'priority' %>
+                    <th><%= t('Priority') %></th>
+                <% end %>
+            </tr>
+        </thead>
+
+        <% @jobs.each do |job_desc| %>
+            <tr>
+                <td><%= job_desc['job'] %></td>
+                <td>
+                    <code class="code-wrap">
+                        <div class="args-extended">
+                            <%= job_desc['args']&.to_json %>
+                            <%= job_desc['kwargs']&.to_json %>
+                        </div>
+                    </code>
+                </td>
+                <td><a href="<%= root_path %>batches/<%= job_desc['pool_wrapper_batch'] %>"><%= job_desc['pool_wrapper_batch'] %></a></td>
+                <% if @pool.order == 'priority' %>
+                    <td><%= job_desc['priority'] %></td>
+                <% end %>
+            </tr>
+        <% end %>
+    </table>
+</div>
+<% end %>
+
+<form class="form-horizontal" action="<%= root_path %>pools/<%= @pool.pid %>" method="post">
+    <%= csrf_tag %>
+    <a class="btn btn-default" href="<%= root_path %>pools"><%= t('GoBack') %></a>
+    <input class="btn btn-danger" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>

data/lib/joblin/batching/compat/sidekiq/web/views/pools.erb

@@ -0,0 +1,47 @@
+<header class="row">
+    <div class="col-sm-5">
+        <h3>
+            <%= t('Pools') %>
+        </h3>
+    </div>
+    <% if @pools.any? && @total_size > @count.to_i %>
+        <div class="col-sm-4">
+            <%= erb get_template(:_pagination), locals: { url: "#{root_path}pools" } %>
+        </div>
+    <% end %>
+</header>
+
+<% if @pools.any? %>
+<div class="table_container">
+    <table class="table table-striped table-bordered table-hover">
+        <thead>
+            <tr>
+                <th><%= t('Created') %></th>
+                <th><%= t('PID') %></th>
+                <th><%= t('Description') %></th>
+                <th><%= t('Type') %></th>
+                <th><%= t('Concurrency') %></th>
+                <th><%= t('Utilization') %></th>
+                <th><%= t('Pending Tasks') %></th>
+            </tr>
+        </thead>
+
+        <% @pools.each do |pool| %>
+        <tr>
+            <td><%= safe_relative_time(pool.created_at.to_f) %></th>
+            <td><a href="<%= root_path %>pools/<%= pool.pid %>"><%= pool.pid %></a></td>
+            <td><%= pool.description %></th>
+            <td><%= pool.order.to_s.upcase %></th>
+            <td><%= pool.concurrency %></th>
+            <td><%= pool.active_count %></th>
+            <td><%= pool.pending_count %></th>
+        </tr>
+        <% end %>
+    </table>
+</div>
+<% end %>
+
+<form action="<%= root_path %>pools/all" method="post">
+    <%= csrf_tag %>
+    <input class="btn btn-danger btn-xs pull-right flip" type="submit" name="delete" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>

data/lib/joblin/batching/compat/sidekiq/web.rb

@@ -0,0 +1,218 @@
+
+begin
+  require "sidekiq/web"
+rescue LoadError
+  # client-only usage
+end
+
+require_relative "web/helpers"
+
+module Joblin::Batching::Compat::Sidekiq
+  module Web
+    DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
+    ::Sidekiq::WebHelpers::SAFE_QPARAMS << 'all_batches'
+    ::Sidekiq::WebHelpers::SAFE_QPARAMS << 'count'
+
+    def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      app.helpers do
+        include Web::Helpers
+
+        def dev_mode?
+          DEV_MODE
+        end
+      end
+
+      # =============== BATCHES =============== #
+
+      app.get "/batches" do
+        @count = (params['count'] || 25).to_i
+
+        source_key = params['all_batches'] ? "batches" : "BID-ROOT-bids"
+        @current_page, @total_size, @batches = page(source_key, params['page'], @count)
+        @batches = @batches.map {|b, score| Joblin::Batching::Batch.new(b) }
+
+        erb(get_template(:batches))
+      end
+
+      app.get "/batches/:bid" do
+        @bid = params[:bid]
+        @batch = Joblin::Batching::Batch.new(@bid)
+
+        @tree_data = tree_data(@bid)
+
+        @count = (params['count'] || 25).to_i
+        @current_batches_page, @total_batches_size, @sub_batches = page("BID-#{@batch.bid}-bids", params['batch_page'], @count)
+        @sub_batches = @sub_batches.map {|b, score| Joblin::Batching::Batch.new(b) }
+
+        @current_jobs_page, @total_jobs_size, @jobs = page("BID-#{@batch.bid}-jids", params['job_page'], @count)
+        @jobs = @jobs.map do |jid, score|
+          { jid: jid, }
+        end
+
+        erb(get_template(:batch))
+      end
+
+      app.get "/batches/:bid/tree" do
+        @bid = params[:bid]
+
+        json(tree_data(@bid, slice: params[:slice]))
+      end
+
+      app.helpers do
+        def tree_data(root_bid, slice: nil)
+          tree_bids = Joblin::Batching::Batch.bid_hierarchy(root_bid, slice: slice)
+
+          Joblin::Batching::Batch.redis do |r|
+            layer_data = ->(layer, parent = nil) {
+              bid = layer[0]
+              batch = Joblin::Batching::Batch.new(bid)
+
+              jobs_total = r.hget("BID-#{bid}", "job_count").to_i
+              jobs_pending = r.hget("BID-#{bid}", 'pending').to_i
+              jobs_failed = r.scard("BID-#{bid}-failed").to_i
+              jobs_dead = r.scard("BID-#{bid}-dead").to_i
+              jobs_success = jobs_total - jobs_pending
+
+              batches_total = r.hget("BID-#{bid}", 'children').to_i
+              batches_success = r.scard("BID-#{bid}-batches-success").to_i
+              batches_pending = batches_total - batches_success
+              batches_failed = r.scard("BID-#{bid}-batches-failed").to_i
+
+              status = 'in_progress'
+              status = 'complete' if batches_pending == batches_failed && jobs_pending == jobs_failed
+              status = 'success' if batches_pending == 0 && jobs_pending == 0
+              status = 'deleted' if bid != root_bid && !batch.parent_bid
+
+              {
+                bid: bid,
+                created_at: r.hget("BID-#{bid}", 'created_at'),
+                status: status,
+                parent_bid: parent ? parent.bid : batch.parent_bid,
+                description: batch.description,
+                jobs: {
+                  pending_count: jobs_pending,
+                  successful_count: jobs_success,
+                  failed_count: jobs_failed,
+                  dead_count: jobs_dead,
+                  total_count: jobs_total,
+                  # items: batches.map{|b| layer_data[b] },
+                },
+                batches: {
+                  pending_count: batches_pending,
+                  successful_count: batches_success,
+                  failed_count: batches_failed,
+                  total_count: batches_total,
+                  items: layer[1].map{|b| layer_data[b, batch] },
+                },
+              }
+            }
+
+            data = layer_data[tree_bids]
+            data[:batches][:slice] = slice if slice
+            data
+          end
+        end
+
+        def format_context(batch)
+          bits = []
+          own_keys = batch.context.own.keys
+          batch.context.flatten.each do |k,v|
+            added = own_keys.include? k
+            bits << "  <span class=\"key #{added ? 'own' : 'inherited'}\">\"#{k}\": #{v.to_json},</span>"
+          end
+          bits = [
+            "{ // <span class=\"own\">Added</span> / <span class=\"inherited\">Inherited</span>",
+            *bits,
+            '}'
+          ]
+          bits.join("\n")
+        end
+      end
+
+      app.post "/batches/all" do
+        if params['delete']
+          index_key = Joblin::Batching::Batch::INDEX_ALL_BATCHES ? "batches" : "BID-ROOT-bids"
+          drain_zset(index_key) do |batches|
+            batches.each do |bid|
+              Joblin::Batching::Batch.cleanup_redis(bid)
+            end
+          end
+        end
+
+        redirect "#{root_path}batches"
+      end
+
+      app.post "/batches/:bid" do
+        @bid = params[:bid]
+        @batch = Joblin::Batching::Batch.new(@bid)
+
+        if params['delete']
+          Joblin::Batching::Batch.delete_prematurely!(@bid)
+        end
+
+        redirect_with_query("#{root_path}batches")
+      end
+
+      # =============== POOLS =============== #
+
+      app.get "/pools" do
+        @count = (params['count'] || 25).to_i
+        @current_page, @total_size, @pools = page('pools', params['page'], @count)
+        @pools = @pools.map {|b, score| Joblin::Batching::Pool.new(b) }
+
+        erb(get_template(:pools))
+      end
+
+      app.get "/pools/:pid" do
+        @pid = params[:pid]
+        @pool = Joblin::Batching::Pool.new(@pid)
+
+        @active_tasks = @pool.active_jobs
+
+        @count = (params['count'] || 25).to_i
+        @current_jobs_page, @total_jobs_size, @jobs = page("POOLID-#{@pool.pid}-jobs", params['job_page'], @count)
+        @jobs = @jobs.map {|desc, score=nil| JSON.parse(desc)[0] }
+
+        erb(get_template(:pool))
+      end
+
+      app.post "/pools/all" do
+        if params['delete']
+          drain_zset('pools') do |pools|
+            pools.each do |pid|
+              Joblin::Batching::Pool.from_pid(pid).cleanup_redis
+            end
+          end
+        end
+
+        redirect "#{root_path}pools"
+      end
+
+      app.post "/pools/:pid" do
+        @pid = params[:pid]
+        @pool = Joblin::Batching::Pool.from_pid(@pid)
+
+        if params['delete']
+          @pool.cleanup_redis
+        end
+
+        redirect_with_query("#{root_path}pools")
+      end
+    end
+  end
+end
+
+if defined?(::Sidekiq::Web)
+  rules = []
+  rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless Joblin::Batching::Compat::Sidekiq::Web::DEV_MODE
+
+  ::Sidekiq::Web.use Rack::Static, urls: ["/batches_assets"],
+                                   root: File.expand_path("#{File.dirname(__FILE__)}/web"),
+                                   cascade: true,
+                                   header_rules: rules
+
+  ::Sidekiq::Web.register Joblin::Batching::Compat::Sidekiq::Web
+  ::Sidekiq::Web.tabs["Batches"] = "batches"
+  ::Sidekiq::Web.tabs["Pools"] = "pools"
+  ::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
+end