joblin 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d79f8417c584523ab669a7e4c4d13df180eaad4c921b915073aec83e2c8b6197
-  data.tar.gz: 334c035490eeb2537a8503df28585ef6cf736c25ca8822e82e812c2f0841a443
+  metadata.gz: 6cb0e0c2038030615d4e6a50e08aac0166f022a6af89f2a5d157d977e2f03f58
+  data.tar.gz: f25744584d54054fe566770b10870159ac9c2a422c8e44cb609aca67116f4c03
 SHA512:
-  metadata.gz: 48b1d7e9370ca38d31a1a4f4f17943210cd3203a2e0e0c397be45878c520e2a7c50653147e1feb6596cf17ab32fdbccd681938b13fbcf528d660fb358282444b
-  data.tar.gz: 7deceab65a315fb8534db3d436fab18ebf96e1452c4d01887a7c35c4b6e57831a6b96299e86f3d3623e1f4b33cab01b78cf9aed664ed6e989b017c28744654da
+  metadata.gz: 726db8abb51ae0697aa6299df1179c7ef7a5e7de96681b6da9341d97242d4a39928ca4a274a26d58b35ba519f29c3b7d18b8d19dc4cf66bbdd7f25f0c5f83a2b
+  data.tar.gz: 83ca76bd6c338207259f08e4115920613c525c5bbe3be431a6a1299bfc0b79282f69a986ce3c04fa17078b0c2dce24f1de1e8556e8d371bda03bcb7231c5ba15

data/app/models/joblin/background_task/api_access.rb

@@ -1,155 +1,159 @@
 module Joblin
-  module BackgroundTask::ApiAccess
-    extend ActiveSupport::Concern
-
-    class_methods do
-      def api_access_rules(&blk)
-        @api_access_rules ||= []
-        if blk
-          @api_access_rules << ApiAccessRules.new.tap do |aar|
-            aar.instance_exec(&blk)
+  class BackgroundTask
+    module ApiAccess
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def api_access_rules(&blk)
+          @api_access_rules ||= []
+          if blk
+            @api_access_rules << ApiAccessRules.new.tap do |aar|
+              aar.instance_exec(&blk)
+            end
+            nil
+          else
+            [*superclass.try(:api_access_rules), *@api_access_rules].compact.freeze
           end
-          nil
-        else
-          [*superclass.try(:api_access_rules), *@api_access_rules].compact.freeze
         end
-      end
 
-      def allow_api_access!(&blk)
-        include Mixin unless self < Mixin
-        @api_access_allowed = true
-        api_access_rules(&blk) if blk
-      end
-
-      def api_access_allowed?
-        # This is intentionally not inherited by subclasses
-        @api_access_allowed
-      end
+        def allow_api_access!(&blk)
+          include Mixin unless self < Mixin
+          @api_access_allowed = true
+          api_access_rules(&blk) if blk
+        end
 
-      def find_for_api(id)
-        task = find(id)
-        raise ActiveRecord::RecordNotFound unless task.api_access_allowed?
-        task
-      end
+        def api_access_allowed?
+          # This is intentionally not inherited by subclasses
+          @api_access_allowed
+        end
 
-      def build_from_api(controller_or_params, options = {}, params: nil, exec_context: nil)
-        if controller_or_params.respond_to?(:params)
-          exec_context ||= controller_or_params
-          params ||= controller_or_params.params.require(:background_task)
-        else
-          params ||= controller_or_params
+        def find_for_api(id)
+          task = find(id)
+          raise ActiveRecord::RecordNotFound unless task.api_access_allowed?
+          task
         end
 
-        task_type = self == BackgroundTask ? params[:type].safe_constantize : self
-        raise ActiveRecord::RecordNotFound unless task_type && task_type <= BackgroundTask
+        def build_from_api(controller_or_params, options = {}, params: nil, exec_context: nil)
+          if controller_or_params.respond_to?(:params)
+            exec_context ||= controller_or_params
+            params ||= controller_or_params.params.require(:background_task)
+          else
+            params ||= controller_or_params
+          end
+
+          task_type = self == BackgroundTask ? params[:type].safe_constantize : self
+          raise ActiveRecord::RecordNotFound unless task_type && task_type <= BackgroundTask
 
-        task = task_type.new
-        raise ActiveRecord::RecordNotFound unless task.api_access_allowed?
-        rule_sets = task_type.api_access_rules
+          task = task_type.new
+          raise ActiveRecord::RecordNotFound unless task.api_access_allowed?
+          rule_sets = task_type.api_access_rules
 
-        # Apply permitted options
-        if params[:options].present?
-          permitted_options = rule_sets.map(&:permitted_options).flatten
-          task.options.merge!(params[:options].permit(permitted_options).to_h)
-        end
+          # Apply permitted options
+          if params[:options].present?
+            permitted_options = rule_sets.map(&:permitted_options).flatten
+            task.options.merge!(params[:options].permit(permitted_options).to_h)
+          end
 
-        # Apply default options
-        rule_sets.reverse.each do |aar|
-          aar.default_options.each do |k, v|
-            next if task.options.key?(k)
-            task.options[k] = v.is_a?(Proc) ? exec_context.instance_exec(&v) : v
+          # Apply default options
+          rule_sets.reverse.each do |aar|
+            aar.default_options.each do |k, v|
+              next if task.options.key?(k)
+              task.options[k] = v.is_a?(Proc) ? exec_context.instance_exec(&v) : v
+            end
           end
-        end
 
-        # Apply any additional/hard-coded options
-        task.options.merge!(options)
+          # Apply any additional/hard-coded options
+          task.options.merge!(options)
 
-        raise ActiveRecord::RecordInvalid.new(task) unless task.api_access_allowed?
+          raise ActiveRecord::RecordInvalid.new(task) unless task.api_access_allowed?
 
-        val_errors = task.api_validate_options
-        raise ActiveRecord::ValidationError.new(val_errors) unless val_errors.empty?
+          val_errors = task.api_validate_options
+          raise ActiveRecord::ValidationError.new(val_errors) unless val_errors.empty?
 
-        task
+          task
+        end
       end
-    end
 
-    included do
-      api_access_rules do
-        default_option(:creator) { try(:current_user) }
+      included do
+        api_access_rules do
+          default_option(:creator) { try(:current_user) }
+        end
       end
-    end
 
-    def api_access_allowed?
-      self.class.api_access_allowed?
-    end
+      def api_access_allowed?
+        self.class.api_access_allowed?
+      end
 
-    module Mixin
-      extend ActiveSupport::Concern
+      module Mixin
+        extend ActiveSupport::Concern
 
-      included do
-        begin
-          ::BackgroundTaskChannel
-          after_commit do
-            ::BackgroundTaskChannel.broadcast_to(self, api_serialize) if api_access_allowed?
+        included do
+          begin
+            ::BackgroundTaskChannel
+            after_commit do
+              ::BackgroundTaskChannel.broadcast_to(self, api_serialize) if api_access_allowed?
+            end
+          rescue NameError
           end
-        rescue NameError
         end
-      end
 
-      def api_serialize
-        builder = Jbuilder.new do |json|
-          json.id id
-          json.type type
-          json.workflow_state workflow_state
+        def api_serialize
+          builder = Jbuilder.new do |json|
+            json.id id
+            json.type type
+            json.workflow_state workflow_state
 
-          rule_sets = self.class.api_access_rules
-          rule_sets.each do |aar|
-            instance_exec(json, &aar.serializer) if aar.serializer
+            json.error_message error_message || "Internal Error" if workflow_state == 'failed'
+
+            rule_sets = self.class.api_access_rules
+            rule_sets.each do |aar|
+              instance_exec(json, &aar.serializer) if aar.serializer
+            end
           end
-        end
 
-        builder.attributes!
-      end
+          builder.attributes!
+        end
 
-      def api_validate_options
-        errors = []
-        rule_sets = self.class.api_access_rules
-        rule_sets.each do |aar|
-          aar.validators.each do |validator|
-            errors << instance_exec(&validator)
+        def api_validate_options
+          errors = []
+          rule_sets = self.class.api_access_rules
+          rule_sets.each do |aar|
+            aar.validators.each do |validator|
+              errors << instance_exec(&validator)
+            end
           end
+          errors.flatten.compact.uniq
         end
-        errors.flatten.compact.uniq
       end
-    end
 
-    class ApiAccessRules
-      attr_accessor :serializer
-      attr_reader :validators
-      attr_reader :default_options
-      attr_reader :permitted_options
-
-      def initialize
-        @serializer = nil
-        @validators = []
-        @permitted_options = []
-        @default_options = {}
-      end
+      class ApiAccessRules
+        attr_accessor :serializer
+        attr_reader :validators
+        attr_reader :default_options
+        attr_reader :permitted_options
+
+        def initialize
+          @serializer = nil
+          @validators = []
+          @permitted_options = []
+          @default_options = {}
+        end
 
-      def default_option(key, value = nil, &blk)
-        @default_options[key] = blk || value
-      end
+        def default_option(key, value = nil, &blk)
+          @default_options[key] = blk || value
+        end
 
-      def permit_options(*args)
-        @permitted_options.concat(args)
-      end
+        def permit_options(*args)
+          @permitted_options.concat(args)
+        end
 
-      def validate(&blk)
-        @validators << blk
-      end
+        def validate(&blk)
+          @validators << blk
+        end
 
-      def serialize(&blk)
-        @serializer = blk
+        def serialize(&blk)
+          @serializer = blk
+        end
       end
     end
   end

data/app/models/joblin/background_task/attachments.rb

@@ -1,49 +1,51 @@
 module Joblin
-  module BackgroundTask::Attachments
-    extend ActiveSupport::Concern
-
-    include Concerns::JobWorkingDirs
-
-    def attachment_path(key, expires_in: true)
-      if !expires_in || Rails.env.development? || Rails.env.test?
-        return Rails.application.routes.url_helpers.rails_blob_path(
-          send(key),
-          only_path: true,
-          disposition: 'attachment',
-          # organization_id: current_organization&.id,
-        )
-      end
+  class BackgroundTask
+    module Attachments
+      extend ActiveSupport::Concern
+
+      include Joblin::Concerns::JobWorkingDirs
+
+      def attachment_path(key, expires_in: true)
+        if !expires_in || Rails.env.development? || Rails.env.test?
+          return Rails.application.routes.url_helpers.rails_blob_path(
+            send(key),
+            only_path: true,
+            disposition: 'attachment',
+            # organization_id: current_organization&.id,
+          )
+        end
 
-      if expires_in == true
-        send(key).url
-      else expires_in
-        send(key).url(expires_in:)
+        if expires_in == true
+          send(key).url
+        else expires_in
+          send(key).url(expires_in:)
+        end
       end
-    end
 
-    def attach_file(key, path, as: nil)
-      path = File.join(working_dir, path) unless Pathname.new(path).absolute?
+      def attach_file(key, path, as: nil)
+        path = File.join(working_dir, path) unless Pathname.new(path).absolute?
 
-      self.send(key).attach(
-        io: File.open(path),
-        filename: as || File.basename(path)
-      )
-    end
+        self.send(key).attach(
+          io: File.open(path),
+          filename: as || File.basename(path)
+        )
+      end
 
-    def load_attachment(key, save_as: nil)
-      @loaded_attachments ||= {}
+      def load_attachment(key, save_as: nil)
+        @loaded_attachments ||= {}
 
-      @loaded_attachments[key] ||= begin
-        save_as = save_as || send(key).filename.to_s || key.to_s
-        save_as = File.join(working_dir, save_as) unless Pathname.new(save_as).absolute?
+        @loaded_attachments[key] ||= begin
+          save_as = save_as || send(key).filename.to_s || key.to_s
+          save_as = File.join(working_dir, save_as) unless Pathname.new(save_as).absolute?
 
-        File.open(save_as, 'w') do |file|
-          send(key).download do |chunk|
-            file.write(chunk.force_encoding("UTF-8"))
+          File.open(save_as, 'w') do |file|
+            send(key).download do |chunk|
+              file.write(chunk.force_encoding("UTF-8"))
+            end
           end
+          save_as
         end
-        save_as
       end
     end
   end
-end
+end

data/app/models/joblin/background_task/executor.rb

@@ -1,79 +1,81 @@
 module Joblin
-  module BackgroundTask::Executor
-    extend ActiveSupport::Concern
+  class BackgroundTask
+    module Executor
+      extend ActiveSupport::Concern
 
-    class_methods do
-      def inherited(subclass)
-        subclass.const_set(:ExecutorJob, Class.new(self::ExecutorJob))
-        super
-      end
+      class_methods do
+        def inherited(subclass)
+          subclass.const_set(:ExecutorJob, Class.new(self::ExecutorJob))
+          super
+        end
 
-      def job_executor_class
-        self::ExecutorJob
-      end
+        def job_executor_class
+          self::ExecutorJob
+        end
 
-      def executor_eval(&blk)
-        job_executor_class.class_eval(&blk)
-      end
+        def executor_eval(&blk)
+          job_executor_class.class_eval(&blk)
+        end
 
-      delegate :queue_as, :sidekiq_options, to: :job_executor_class
+        delegate :queue_as, :sidekiq_options, to: :job_executor_class
 
-      %i[before around after].each do |hook_type|
-        define_method(:"#{hook_type}_perform") do |*args, &callback|
-          callback ||= args[0]
-          if callback.is_a?(Symbol) || callback.is_a?(String)
-            job_executor_class.define_method(callback) do |*margs, &blk|
-              the_task.send(callback, *margs, &blk)
-            end
-            job_executor_class.send(:"#{hook_type}_perform", callback)
-          elsif callback.is_a?(Proc)
-            # Not exactly pretty...
-            proc = case callback.arity
-            when 0
-              -> { the_task.instance_exec(&callback) }
-            when 1
-              ->(a) { the_task.instance_exec(a, &callback) }
-            when 2
-              ->(a, b) { the_task.instance_exec(a, b, &callback) }
-            else
-              raise ArgumentError, "Unsupported callback arity #{callback.arity}"
+        %i[before around after].each do |hook_type|
+          define_method(:"#{hook_type}_perform") do |*args, &callback|
+            callback ||= args[0]
+            if callback.is_a?(Symbol) || callback.is_a?(String)
+              job_executor_class.define_method(callback) do |*margs, &blk|
+                the_task.send(callback, *margs, &blk)
+              end
+              job_executor_class.send(:"#{hook_type}_perform", callback)
+            elsif callback.is_a?(Proc)
+              # Not exactly pretty...
+              proc = case callback.arity
+              when 0
+                -> { the_task.instance_exec(&callback) }
+              when 1
+                ->(a) { the_task.instance_exec(a, &callback) }
+              when 2
+                ->(a, b) { the_task.instance_exec(a, b, &callback) }
+              else
+                raise ArgumentError, "Unsupported callback arity #{callback.arity}"
+              end
+              job_executor_class.send(:"#{hook_type}_perform", &proc)
             end
-            job_executor_class.send(:"#{hook_type}_perform", &proc)
           end
         end
-      end
-
-      # delegate :set, to: :job_executor_class
-
-      # delegate :before_perform, :around_perform, :after_perform, to: :job_executor_class
-      # delegate :before_enqueue, :around_enqueue, :after_enqueue, to: :job_executor_class
-    end
-
-    included do
-      delegate_missing_to :_current_executor
-    end
 
-    protected
+        # delegate :set, to: :job_executor_class
 
-    def _current_executor
-      @executor
-    end
+        # delegate :before_perform, :around_perform, :after_perform, to: :job_executor_class
+        # delegate :before_enqueue, :around_enqueue, :after_enqueue, to: :job_executor_class
+      end
 
-    class ExecutorJob < ActiveJob::Base
-      # include JobWorkingDirs
+      included do
+        delegate_missing_to :_current_executor
+      end
 
-      def perform
-        the_task.update(workflow_state: "started") if the_task.workflow_state == "scheduled"
+      protected
 
-        the_task.perform
+      def _current_executor
+        @executor
       end
 
-      def the_task
-        @the_task ||= BackgroundTask.find(batch_context[:background_task_id]).tap do |task|
-          task.instance_variable_set(:@executor, self)
+      class ExecutorJob < ActiveJob::Base
+        def perform
+          the_task.update(workflow_state: "started") if the_task.workflow_state == "scheduled"
+          the_task.perform
+        rescue InvalidJobError => ex
+          the_task.error_message ||= ex.message
+          the_task.workflow_state = "failed"
+          the_task.save! if the_task.changed?
+        end
+
+        def the_task
+          @the_task ||= BackgroundTask.find(batch_context[:background_task_id]).tap do |task|
+            task.instance_variable_set(:@executor, self)
+          end
         end
       end
     end
-
   end
 end

data/app/models/joblin/background_task/options.rb

@@ -1,5 +1,5 @@
-module Joblin
-  module BackgroundTask::Options
+class Joblin::BackgroundTask
+  module Options
     extend ActiveSupport::Concern
 
     class_methods do
@@ -16,7 +16,8 @@ module Joblin
     end
 
     included do
-      serialize :extra_options, coder: LazyAccess
+      serialize :results, coder: Joblin::LazyAccess
+      serialize :extra_options, coder: Joblin::LazyAccess
 
       after_initialize do
         self.extra_options ||= HashWithIndifferentAccess.new

data/app/models/joblin/background_task/retention_policy.rb

@@ -1,26 +1,28 @@
 module Joblin
-  module BackgroundTask::RetentionPolicy
-    extend ActiveSupport::Concern
+  class BackgroundTask
+    module RetentionPolicy
+      extend ActiveSupport::Concern
 
-    class_methods do
-      def record_retention(policy = nil)
-        if policy.nil?
-          @record_retention || superclass.try(:record_retention)
-        else
-          @record_retention = policy
+      class_methods do
+        def record_retention(policy = nil)
+          if policy.nil?
+            @record_retention || superclass.try(:record_retention)
+          else
+            @record_retention = policy
+          end
         end
       end
-    end
 
-    class BackgroundTaskCleaner < ActiveJob::Base
-      def perform
-        types = BackgroundTask.distinct.pluck(:type)
-        types.each do |type|
-          type = type.constantize
-          rp = type.record_retention
-          next unless rp
+      class BackgroundTaskCleaner < ActiveJob::Base
+        def perform
+          types = BackgroundTask.distinct.pluck(:type)
+          types.each do |type|
+            type = type.constantize
+            rp = type.record_retention
+            next unless rp
 
-          type.where("created_at < ?", rp.ago).destroy_all
+            type.where("created_at < ?", rp.ago).destroy_all
+          end
         end
       end
     end

data/app/models/joblin/background_task.rb

@@ -6,8 +6,12 @@ module Joblin
     include Attachments
     include ApiAccess
 
+    class InvalidJobError < StandardError; end
+
     belongs_to :creator, polymorphic: true, optional: true
 
+    validates :workflow_state, inclusion: { in: %w[unscheduled scheduled started completed failed cancelled] }
+
     after_initialize do
       self.workflow_state ||= 'unscheduled'
     end
@@ -34,6 +38,10 @@ module Joblin
       update!(workflow_state: 'cancelled')
     end
 
+    def handle_batch_stagnation
+      Joblin::Batching::Batch.cleanup_redis(batch_id)
+    end
+
     protected
 
     def perform
@@ -46,6 +54,7 @@ module Joblin
         b.description = "BackgroundTask #{id}"
         b.on(:complete, BackgroundTaskCallbacks, id: id)
         b.on(:success, BackgroundTaskCallbacks, id: id)
+        b.on(:stagnated, BackgroundTaskCallbacks, id: id)
         b.context[:background_task_id] = id
       end
       b.jobs(&blk)
@@ -56,16 +65,21 @@ module Joblin
         tracker = BackgroundTask.find(options['id'])
         return if tracker.workflow_state == 'cancelled'
 
-        tracker.workflow_state = status.success? ? 'completed' : 'failed'
-        tracker.save! if tracker.changed?
+        # tracker.update!(workflow_state: 'completed') if status.success?
       end
 
       def on_success(status, options)
         tracker = BackgroundTask.find(options['id'])
         return if tracker.workflow_state == 'cancelled'
+        return if tracker.workflow_state == 'failed'
 
-        tracker.workflow_state = 'completed'
-        tracker.save! if tracker.changed?
+        tracker.update!(workflow_state: 'completed')
+      end
+
+      def on_stagnated(status, options)
+        tracker = BackgroundTask.find(options['id'])
+        tracker.update(workflow_state: 'failed') if tracker.workflow_state != 'cancelled'
+        tracker.handle_batch_stagnation
       end
     end
   end

data/db/migrate/20250916184852_add_results_to_joblin_background_tasks.rb

@@ -0,0 +1,6 @@
+class AddResultsToJoblinBackgroundTasks < ActiveRecord::Migration[5.2]
+  def change
+    add_column :joblin_background_tasks, :results, :jsonb
+    add_column :joblin_background_tasks, :error_message, :text
+  end
+end

data/lib/joblin/version.rb

@@ -1,3 +1,3 @@
 module Joblin
-  VERSION = "0.1.3".freeze
+  VERSION = "0.1.5".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: joblin
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Ethan Knapp
@@ -156,6 +156,7 @@ files:
 - app/models/joblin/background_task/retention_policy.rb
 - app/models/joblin/concerns/job_working_dirs.rb
 - db/migrate/20250903184852_create_joblin_background_tasks.rb
+- db/migrate/20250916184852_add_results_to_joblin_background_tasks.rb
 - joblin.gemspec
 - lib/joblin.rb
 - lib/joblin/batching/batch.rb