joblin 0.1.0

data/lib/joblin/batching/chain_builder.rb

@@ -0,0 +1,247 @@
+module Joblin::Batching
+  class ChainBuilder
+    VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
+
+    attr_reader :base_job
+
+    def initialize(base_type = SerialBatchJob)
+      if base_type.is_a?(Hash)
+        @base_job = base_type
+        @base_job[:args] ||= @base_job[:parameters] || []
+        @base_job[:kwargs] ||= {}
+      else
+        @base_job = build_job_hash(base_type)
+      end
+
+      self.class.get_chain_parameter(base_job)
+    end
+
+    def process!
+      normalize!
+      self.class.enqueue_job(base_job)
+    end
+
+    def [](key)
+      if key.is_a?(Class)
+        get_sub_chain(key)
+      else
+        # Legacy Support
+        key = :args if key == :parameters
+
+        @base_job[key]
+      end
+    end
+
+    def args; return self[:args]; end
+    def kwargs; return self[:kwargs]; end
+
+    def <<(new_job)
+      insert_at(-1, new_job)
+    end
+
+    def insert_at(position, new_jobs, *args, **kwargs, &blk)
+      chain = self.class.get_chain_parameter(base_job)
+      if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
+        new_jobs = build_job_hash(new_jobs, args: args, kwargs: kwargs, &blk)
+      elsif args.count > 0 || kwargs.count > 0
+        raise "Unexpected number of arguments"
+      end
+      new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
+      chain.insert(position, *new_jobs)
+    end
+
+    def insert(new_jobs, *args, **kwargs, &blk)
+      if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
+        job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
+        new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
+        kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
+      else
+        invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
+        raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
+        raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
+        raise "Unexpected number of arguments" if args.length > 0
+      end
+
+      new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
+
+      if !kwargs.present?
+        insert_at(-1, new_jobs)
+      else
+        placement = kwargs.keys[0]
+        relative_to = kwargs.values[0]
+
+        matching_jobs = find_matching_jobs(relative_to).to_a
+        raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
+        raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
+
+        relative_job, parent_job, sub_index = matching_jobs[0]
+        needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
+
+        chain = self.class.get_chain_parameter(parent_job)
+
+        if parent_job[:job] != needed_parent_type
+          old_job = chain[sub_index]
+          parent_job = chain[sub_index] = {
+            job: needed_parent_type,
+            parameters: [],
+          }
+          sub_index = 0
+          chain = self.class.get_chain_parameter(parent_job)
+          chain << old_job
+        end
+
+        if placement == :with
+          chain.insert(-1, *new_jobs)
+        else
+          sub_index += 1 if placement == :after
+          chain.insert(sub_index, *new_jobs)
+        end
+      end
+    end
+
+    def empty?
+      self.class.get_chain_parameter(self).empty?
+    end
+
+    def get_sub_chain(sub_type)
+      matching_jobs = find_matching_jobs(sub_type).to_a
+      raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
+      return nil if matching_jobs.count == 0
+
+      job = matching_jobs[0][0]
+      job = self.class.new(job) unless job.is_a?(ChainBuilder)
+      job
+    end
+
+    def normalize!(job_def = self.base_job)
+      if job_def.is_a?(ChainBuilder)
+        job_def.normalize!
+      else
+        job_def[:job] = job_def[:job].to_s
+        if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
+          chain.map! { |sub_job| normalize!(sub_job) }
+        end
+        job_def
+      end
+    end
+
+    def apply_block(&blk)
+      return unless blk.present?
+      instance_exec(&blk)
+    end
+
+    private
+
+    def build_job_hash(job, args: [], kwargs: {}, &blk)
+      hsh = {
+        job: job,
+        args: args,
+        kwargs: kwargs,
+      }
+      self.class.new(hsh).apply_block(&blk) if blk.present?
+      hsh
+    end
+
+    def find_matching_jobs(search_job, parent_job = self.base_job)
+      return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
+
+      sub_jobs = self.class.get_chain_parameter(parent_job)
+      sub_jobs.each_with_index do |sub_job, i|
+        if sub_job[:job].to_s == search_job.to_s
+          yield [sub_job, parent_job, i]
+        elsif self.class._job_type_definitions[sub_job[:job].to_s]
+          find_matching_jobs(search_job, sub_job) { |item| yield item }
+        end
+      end
+    end
+
+    def find_parent_job(job_def)
+      iterate_job_tree do |job, path|
+        return path[-1] if job == job_def
+      end
+      nil
+    end
+
+    def iterate_job_tree(root: self.base_job, path: [], &blk)
+      blk.call(root, path)
+
+      if self.class._job_type_definitions[root[:job]]
+        sub_jobs = self.class.get_chain_parameter(root)
+        sub_jobs.each_with_index do |sub_job, i|
+          iterate_job_tree(root: sub_job, path: [*path, root], &blk)
+        end
+      end
+    end
+
+    class << self
+      # Support builder syntaxt/DSL
+      # Chain.build(ConcurrentBatchJob) do
+      #   insert(SomeJob, arg1, kwarg: 1)
+      #   insert(SerialBatchJob) do
+      #     insert(SomeJob, arg1, kwarg: 1)
+      #   end
+      # end
+      def build(job, *args, **kwargs, &blk)
+        new(job).tap do |ch|
+          ch.base_job[:args] = args
+          ch.base_job[:kwargs] = kwargs
+          ch.apply_block(&blk)
+        end
+      end
+
+      def _job_type_definitions
+        @job_type_definitions ||= {}
+      end
+
+      def register_chain_job(job_class, chain_parameter, **options)
+        _job_type_definitions[job_class.to_s] = {
+          **options,
+          chain_parameter: chain_parameter,
+        }
+      end
+
+      def get_chain_parameter(job_def, raise_error: true)
+        unless _job_type_definitions[job_def[:job].to_s].present?
+          raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
+          return nil
+        end
+
+        key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
+        if key.is_a?(Numeric)
+          job_def[:args][key] ||= []
+        else
+          job_def[:kwargs][key] ||= []
+        end
+      end
+
+      # TODO: Add a Chain progress web View
+      # Augment Batch tree-view with Chain data
+      #   > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
+      #   > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
+      #     > Chain-jobs will be supplied chain_id and chain_step_id metadata
+      #     > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
+      #     > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
+      #   > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
+      def enqueue_job(job_def)
+        job_class = job_def[:job].constantize
+        job_args = job_def[:args] || job_def[:parameters] || []
+        job_kwargs = job_def[:kwargs] || {}
+
+        # Legacy Support
+        if job_def[:options]
+          job_args << {} unless job_args[-1].is_a?(Hash)
+          job_args[-1].merge!(job_def[:options])
+        end
+
+        if job_class.respond_to? :perform_async
+          job_class.perform_async(*job_args, **job_kwargs)
+        else
+          job_class.perform_later(*job_args, **job_kwargs)
+        end
+      end
+    end
+  end
+
+  ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
+  ChainBuilder.register_chain_job(SerialBatchJob, 0)
+end

data/lib/joblin/batching/compat/active_job.rb

@@ -0,0 +1,108 @@
+
+module Joblin::Batching
+  module Compat
+    module ActiveJob
+      module BatchAwareJob
+        extend ActiveSupport::Concern
+
+        included do
+          around_perform do |job, block|
+            if (@bid) # This _must_ be @bid - not just bid
+              prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
+              begin
+                Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(@bid)
+                block.call
+                Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
+                Batch.process_successful_job(@bid, job_id)
+              rescue
+                Batch.process_failed_job(@bid, job_id)
+                raise
+              ensure
+                Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
+              end
+            else
+              block.call
+            end
+          end
+
+          around_enqueue do |job, block|
+            if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY])
+              @bid = batch.bid
+              batch.append_jobs(job_id) if @bid
+            end
+            block.call
+          end
+        end
+
+        def bid
+          @bid || Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
+        end
+
+        def batch
+          Thread.current[CURRENT_BATCH_THREAD_KEY]
+        end
+
+        def batch_context
+          batch&.context || {}
+        end
+
+        def valid_within_batch?
+          batch.valid?
+        end
+
+        def serialize
+          super.tap do |data|
+            data['batch_id'] = @bid # This _must_ be @bid - not just bid
+            data
+          end
+        end
+
+        def deserialize(data)
+          super
+          @bid = data['batch_id']
+        end
+      end
+
+      class ActiveJobCallbackWorker < ::ActiveJob::Base
+        include Batch::Callback::CallbackWorkerCommon
+
+        def self.enqueue_all(args, queue)
+          args.each do |arg_set|
+            set(queue: queue).perform_later(*arg_set)
+          end
+        end
+      end
+
+      def self.handle_job_death(job, error = nil)
+        if job.is_a?(Array)
+          event = ActiveSupport::Notifications::Event.new(*job)
+          payload = event.payload
+          job = payload[:job].serialize
+          error = payload[:error]
+        end
+
+        if job["job_id"].present? && job["batch_id"].present?
+          Joblin::Batching::Batch.process_dead_job(job['batch_id'], job['job_id'])
+        end
+      end
+
+      def self.configure
+        ::ActiveJob::Base.include BatchAwareJob
+
+        begin
+          ActiveSupport::Notifications.subscribe "discard.active_job" do |*args|
+            handle_job_death(args)
+          end
+
+          ActiveSupport::Notifications.subscribe "retry_stopped.active_job" do |*args|
+            handle_job_death(args)
+          end
+        rescue => err
+          Rails.logger.warn(err)
+        end
+
+        Batch::Callback.worker_class ||= ActiveJobCallbackWorker
+      end
+    end
+  end
+end

data/lib/joblin/batching/compat/sidekiq/web/batches_assets/css/styles.less

@@ -0,0 +1,182 @@
+
+@color-green: #25c766;
+@color-darkred: #8f0000;
+@color-red: #e03963;
+@color-yellow: #c4c725;
+
+.code-wrap.batch-context .args-extended {
+    white-space: pre;
+
+    .key {
+        white-space: pre-wrap;
+        margin-left: 2em;
+        text-indent: -2em;
+        display: inline-block;
+    }
+
+    .own {
+        color: @color-green;
+    }
+}
+
+
+.batch-tree {
+    .status-block {
+        .tree-stat {
+            margin: 0 4px;
+    
+            &.pending {
+                color: @color-yellow;
+            }
+            &.failed {
+                color: @color-red;
+            }
+            &.dead {
+                color: @color-darkred;
+            }
+            &.success {
+                color: @color-green;
+            }
+            &.total {
+    
+            }
+        }
+    }
+
+    .text-inactive {
+        color: darken(#fff, 35%);
+        font-size: 80%;
+    }
+
+    .tree-header {
+        position: relative;
+
+        .status-block {
+            position: absolute;
+            bottom: 0;
+            width: 100%;
+
+            margin-right: 8px;
+            font-size: 90%;
+            text-align: right;
+
+            .tree-stat {
+                font-style: italic;
+            }
+        }
+    }
+
+    .tree-entry {
+        > .header {
+            display: flex;
+            align-items: center;
+
+            .header-inner {
+                padding: 4px 0;
+                border-bottom: 1px dashed white;
+                display: flex;
+                align-items: center;
+                flex: 1;
+            }
+
+            &:hover {
+                background-color: rgba(0,0,0,0.20);
+                border-radius: 3px;
+            }
+
+            .row-toggle {
+                width: 16px;
+                height: 16px;
+                text-align: center;
+                align-self: center;
+                border-radius: 50%;
+                border: 1px solid #999;
+                text-decoration: none;
+                margin: 0 4px;
+                font-size: 16px;
+                line-height: 15px;
+
+                &.not_applicable {
+                    opacity: 0;
+                    pointer-events: none;
+                }
+            }
+
+            .main {
+                flex: 1;
+                display: flex;
+                align-items: baseline;
+
+                .bid {
+                    font-family: monospace;
+                    padding: 3px 6px;
+                    background: rgba(0,0,0,0.2);
+                    border-radius: 3px;
+                    font-size: 12px;
+                    margin: 0 12px 0 0;
+
+                    &:hover {
+                        .bid-goto {
+                            display: inline-block;
+                            padding: 0 0 0 4px;
+                            font-size: 200%;
+                            line-height: 10px;
+                            vertical-align: sub;
+                            text-decoration: dotted;
+                        }
+                    }
+
+                    .bid-goto {
+                        display: none;
+                    }
+                }
+            }
+
+            .goto-link {
+                margin: 0 8px;
+                display: inline-block;
+                height: 16px;
+                font-size: 90%;
+                border-bottom: 1px dotted white;
+            }
+
+            .status-label {
+                font-family: monospace;
+                padding: 3px 6px;
+                background: rgba(0,0,0,0.2);
+                border-radius: 3px;
+                font-size: 12px;
+                margin: 0 12px 0 0;
+
+                &.deleted {
+                    background: #99999933;
+                }
+                &.failed, &.complete {
+                    background: #99000033;
+                }
+                &.success {
+                    background: #00990033;
+                }
+            }
+
+            .status-block {
+                width: 12em;
+                text-align: center;
+            }
+        }
+
+        > .subitems {
+            padding-left: 16px;
+
+            >.load-more {
+                padding: 4px 0;
+                text-align: center;
+                border-bottom: 1px dashed white;
+                a {
+                    border-bottom: 1px dotted white;
+                    text-decoration: none;
+                }
+            }
+        }
+    }
+}

data/lib/joblin/batching/compat/sidekiq/web/batches_assets/js/batch_tree.js

@@ -0,0 +1,108 @@
+import { h, Component, render } from 'https://esm.sh/preact';
+import htm from 'https://esm.sh/htm';
+import { root_url } from './util.js';
+
+// Initialize htm with Preact
+const html = htm.bind(h);
+
+const StatusBlock = (props) => html`
+  <div class="status-block ${props.class || ''}">
+    ${props.title && props.title + ':'}
+    <span class="tree-stat pending">${props.pending_count}</span>
+    |
+    <span class="tree-stat failed">${props.failed_count}</span>
+    |
+    <span class="tree-stat dead">${props.dead_count}</span>
+    |
+    <span class="tree-stat success">${props.successful_count}</span>
+    /
+    <span class="tree-stat total">${props.total_count}</span>
+  </div>
+`
+
+class TreeLevel extends Component {
+  get bid() {
+    return this.props.data.bid;
+  }
+
+  get batch() {
+    return this.props.data;
+  }
+
+  load_more = async (event) => {
+    event.preventDefault();
+    const l = this.batch.batches.items.length;
+    const resp = await fetch(`${root_url}batches/${this.bid}/tree?slice=${l}:${l + 5 - 1}`)
+    const result = await resp.json()
+    const newEntries = result.batches.items;
+    for (let ent of newEntries) {
+      this.batch.batches.items.push(ent)
+    }
+    this.forceUpdate()
+  }
+
+  toggle = (event) => {
+    event.preventDefault();
+    this.setState({ collapsed: !this.state.collapsed })
+  }
+
+  render() {
+    const { data: bd } = this.props;
+
+    let sub_entries = [];
+    let sub_batches = bd.batches.items;
+    for (let b of sub_batches) {
+      sub_entries.push(html`<${TreeLevel} data=${b} />`)
+    }
+
+    let fully_loaded = !(sub_batches.length < bd.batches.total_count);
+
+    const load_more_elem = html`<div class="load-more">
+      ${sub_entries.length} / ${bd.batches.total_count} Items Loaded - <a href="#" onClick=${this.load_more}>Load More</a>
+    </div>`
+
+    return html`<div class="tree-entry tree-batch">
+      <div class="header">
+        <a class="row-toggle ${!bd.batches.total_count && 'not_applicable'}" onClick=${this.toggle} href="#">
+          ${this.state.collapsed ? '+' : '-'}
+        </a>
+
+        <div class="header-inner">
+          <div class="main">
+            <span class="bid">
+              ${bd.bid}
+              <a class="bid-goto" href="${root_url}batches/${bd.bid}">⇢</a>
+            </span>
+            ${bd.description || (bd.status == 'deleted' && html`<i class="text-inactive">Deleted</i>`) || html`<i class="text-inactive">No Description</i>`}
+          </div>
+
+          <span class="status-label ${bd.status}">${bd.status}</span>
+
+          ${bd.status != 'deleted' && html`
+            <${StatusBlock} class="job-status" title="Jobs" ...${bd.jobs} />
+            <${StatusBlock} class="batch-status" title="Batches" ...${bd.batches} />
+          `}
+        </div>
+      </div>
+
+      <div class="subitems ${this.state.collapsed ? 'hidden' : ''}">
+        ${sub_entries}
+        ${!fully_loaded && load_more_elem}
+      </div>
+    </div>`
+  }
+}
+
+class TreeRoot extends Component {
+  render() {
+    const tree_data = JSON.parse(document.querySelector('#batch-tree #initial-data').innerHTML);
+    return html`
+      <div class="tree-header">
+        <${StatusBlock} pending_count="pending" failed_count="failed" dead_count="dead" successful_count="successful" total_count="total" />
+      </div>
+      <${TreeLevel} data=${tree_data} />
+    `;
+  }
+}
+
+render(html`<${TreeRoot} />`, document.querySelector('#batch-tree'));

data/lib/joblin/batching/compat/sidekiq/web/batches_assets/js/util.js

@@ -0,0 +1,2 @@
+
+export const root_url = (document.querySelector('meta[name="sidekiq-baseurl"]')).content;

data/lib/joblin/batching/compat/sidekiq/web/helpers.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Joblin::Batching::Compat::Sidekiq
+  module Web
+    module Helpers
+      VIEW_PATH    = File.expand_path("../web/views", __dir__)
+
+      module_function
+
+      def get_template(name)
+        File.open(File.join(VIEW_PATH, "#{name}.erb")).read
+      end
+
+      def drain_zset(key)
+        items, _ = Joblin::Batching::Batch.redis do |r|
+          r.multi do |r|
+            r.zrange(key, 0, -1)
+            r.zremrangebyrank(key, 0, -1)
+          end
+        end
+        yield items
+      end
+
+      def safe_relative_time(time)
+        time = parse_time(time)
+        relative_time(time)
+      end
+
+      def parse_time(time)
+        case time
+        when Time
+          time
+        when Integer, Float
+          Time.at(time)
+        else
+          Time.parse(time.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/joblin/batching/compat/sidekiq/web/views/_batch_tree.erb

@@ -0,0 +1,6 @@
+
+<div id="batch-tree" class="batch-tree">
+    <script id="initial-data" type="application/json"><%= @tree_data.to_json %></script>
+</div>
+
+<script type="module" src="<%= root_path %>batches_assets/js/batch_tree.js"></script>