joblin 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 35d34b0f27040ddb94b3cfcd15d01c6fcdc12b18895a6b525032ec1d2d0aa41f
+  data.tar.gz: 6cbc121c07c836a85e77c294c92baf82d3338fac4625f38d697973fdf5a97577
+SHA512:
+  metadata.gz: aa4cad19edc21301edd5ff875bbfbff1fec1eed88a10814112380b14ce3f59f7efa3bd7d82741ed86674df1f48dc765d4df62495bb9c14d0aeb444375893755a
+  data.tar.gz: a59dbee1edea55c4f68ff30607aa20c70862462db1c123a0585ba8f757bfd41440a827b6226b7613d197ab224db71e1a05080cfa43f7d18c7fe15eb6b700cf1c

data/README.md

@@ -0,0 +1 @@
+# Joblin

data/app/models/joblin/background_task/api_access.rb

@@ -0,0 +1,148 @@
+module Joblin
+  module BackgroundTask::ApiAccess
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def api_access_rules(&blk)
+        @api_access_rules ||= []
+        if blk
+          @api_access_rules << ApiAccessRules.new.tap do |aar|
+            aar.instance_exec(&blk)
+          end
+          nil
+        else
+          [*superclass.try(:api_access_rules), *@api_access_rules].compact.freeze
+        end
+      end
+
+      def allow_api_access!(&blk)
+        include ApiAccess::Mixin unless self < ApiAccess::Mixin
+        @api_access_allowed = true
+        api_access_rules(&blk) if blk
+      end
+
+      def api_access_allowed?
+        # This is intentionally not inherited by subclasses
+        @api_access_allowed
+      end
+
+      def find_for_api(id)
+        task = find(id)
+        raise ActiveRecord::RecordNotFound unless task.class.api_access_allowed?
+        task
+      end
+
+      def build_from_api(controller_or_params, options = {}, params: nil, exec_context: nil)
+        if controller_or_params.respond_to?(:params)
+          exec_context ||= controller_or_params
+          params ||= controller_or_params.params.require(:background_task)
+        else
+          params ||= controller_or_params
+        end
+
+        task_type = self == BackgroundTask ? params[:type].safe_constantize : self
+        raise ActiveRecord::RecordNotFound unless task_type && task_type <= BackgroundTask
+        raise ActiveRecord::RecordNotFound unless task_type.api_access_allowed?
+
+        task = task_type.new
+        rule_sets = task_type.api_access_rules
+
+        # Apply permitted options
+        if params[:options].present?
+          permitted_options = rule_sets.map(&:permitted_options).flatten
+          task.options.merge!(params[:options].permit(permitted_options).to_h)
+        end
+
+        # Apply default options
+        rule_sets.reverse.each do |aar|
+          aar.default_options.each do |k, v|
+            next if task.options.key?(k)
+            task.options[k] = v.is_a?(Proc) ? exec_context.instance_exec(&v) : v
+          end
+        end
+
+        # Apply any additional/hard-coded options
+        task.options.merge!(options)
+
+        val_errors = task.api_validate_options
+        raise ActiveRecord::ValidationError.new(val_errors) unless val_errors.empty?
+
+        task
+      end
+    end
+
+    included do
+      api_access_rules do
+        default_option(:creator) { try(:current_user) }
+      end
+    end
+
+    module Mixin
+      extend ActiveSupport::Concern
+
+      included do
+        if defined?(BackgroundTaskChannel)
+          after_commit do
+            BackgroundTaskChannel.broadcast_to(self, api_serialize)
+          end
+        end
+      end
+
+      def api_serialize
+        builder = Jbuilder.new do |json|
+          json.id id
+          json.type type
+          json.workflow_state workflow_state
+
+          rule_sets = self.class.api_access_rules
+          rule_sets.each do |aar|
+            instance_exec(json, &aar.serializer) if aar.serializer
+          end
+        end
+
+        builder.attributes!
+      end
+
+      def api_validate_options
+        errors = []
+        rule_sets = self.class.api_access_rules
+        rule_sets.each do |aar|
+          aar.validators.each do |validator|
+            errors << instance_exec(&validator)
+          end
+        end
+        errors.flatten.compact.uniq
+      end
+    end
+
+    class ApiAccessRules
+      attr_accessor :serializer
+      attr_reader :validators
+      attr_reader :default_options
+      attr_reader :permitted_options
+
+      def initialize
+        @serializer = nil
+        @validators = []
+        @permitted_options = []
+        @default_options = {}
+      end
+
+      def default_option(key, value = nil, &blk)
+        @default_options[key] = blk || value
+      end
+
+      def permit_options(*args)
+        @permitted_options.concat(args)
+      end
+
+      def validate(&blk)
+        @validators << blk
+      end
+
+      def serialize(&blk)
+        @serializer = blk
+      end
+    end
+  end
+end

data/app/models/joblin/background_task/attachments.rb

@@ -0,0 +1,47 @@
+module Joblin
+  module BackgroundTask::Attachments
+    extend ActiveSupport::Concern
+
+    def attachment_path(key, expires_in: true)
+      if !expires_in || Rails.env.development? || Rails.env.test?
+        return Rails.application.routes.url_helpers.rails_blob_path(
+          send(key),
+          only_path: true,
+          disposition: 'attachment',
+          # organization_id: current_organization&.id,
+        )
+      end
+
+      if expires_in == true
+        send(key).url
+      else expires_in
+        send(key).url(expires_in:)
+      end
+    end
+
+    def attach_file(key, path, as: nil)
+      path = File.join(working_dir, path) unless Pathname.new(path).absolute?
+
+      self.send(key).attach(
+        io: File.open(path),
+        filename: as || File.basename(path)
+      )
+    end
+
+    def load_attachment(key, save_as: nil)
+      @loaded_attachments ||= {}
+
+      @loaded_attachments[key] ||= begin
+        save_as = save_as || send(key).filename.to_s || key.to_s
+        save_as = File.join(working_dir, save_as) unless Pathname.new(save_as).absolute?
+
+        File.open(save_as, 'w') do |file|
+          send(key).download do |chunk|
+            file.write(chunk.force_encoding("UTF-8"))
+          end
+        end
+        save_as
+      end
+    end
+  end
+end

data/app/models/joblin/background_task/executor.rb

@@ -0,0 +1,63 @@
+module Joblin
+  module BackgroundTask::Executor
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def inherited(subclass)
+        subclass.const_set(:ExecutorJob, Class.new(self::ExecutorJob))
+        super
+      end
+
+      def job_executor_class
+        self::ExecutorJob
+      end
+
+      def executor_eval(&blk)
+        job_executor_class.class_eval(&blk)
+      end
+
+      delegate :queue_as, :sidekiq_options, to: :job_executor_class
+
+      %i[before around after].each do |hook_type|
+        define_method(:"#{hook_type}_perform") do |*args, &blk|
+          blk ||= args[0].to_proc
+          job_executor_class.send(:"#{hook_type}_perform") do |*args|
+            the_task.instance_exec(&blk)
+          end
+        end
+      end
+
+      # delegate :set, to: :job_executor_class
+
+      # delegate :before_perform, :around_perform, :after_perform, to: :job_executor_class
+      # delegate :before_enqueue, :around_enqueue, :after_enqueue, to: :job_executor_class
+    end
+
+    included do
+      delegate_missing_to :_current_executor
+    end
+
+    protected
+
+    def _current_executor
+      @executor
+    end
+
+    class ExecutorJob < ActiveJob::Base
+      # include JobWorkingDirs
+
+      def perform
+        the_task.update(workflow_state: "started") if the_task.workflow_state == "scheduled"
+
+        the_task.perform
+      end
+
+      def the_task
+        @the_task ||= BackgroundTask.find(batch_context[:background_task_id]).tap do |task|
+          task.instance_variable_set(:@executor, self)
+        end
+      end
+    end
+
+  end
+end

data/app/models/joblin/background_task/options.rb

@@ -0,0 +1,75 @@
+module Joblin
+  module BackgroundTask::Options
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def bt_passthrough_values
+        @bt_passthrough_values ||= (instance_methods - ActiveRecord::Base.instance_methods - %i[options= batch_id= id_value= type=]).map(&:to_s).select { |m| m.end_with?('=') }.map { |m| m[0..-2] }
+      end
+    end
+
+    included do
+      serialize :extra_options, coder: LazyAccess
+
+      after_initialize do
+        self.extra_options ||= HashWithIndifferentAccess.new
+      end
+    end
+
+    def options
+      @task_options_view ||= TaskOptionsView.new(self)
+    end
+
+    class TaskOptionsView
+      def initialize(context)
+        @context = context
+      end
+
+      def [](key)
+        if @context.class.bt_passthrough_values.include?(key.to_s)
+          @context.public_send(key.to_sym)
+        else
+          inner_hash[key]
+        end
+      end
+
+      def []=(key, value)
+        if @context.class.bt_passthrough_values.include?(key.to_s)
+          @context.public_send("#{key.to_sym}=", value)
+        else
+          inner_hash[key] = value
+        end
+      end
+
+      def merge(other); to_h.merge!(other); end
+
+      def merge!(other)
+        other.each do |k, v|
+          self[k] = v
+        end
+      end
+
+      def keys
+        inner_hash.keys + @context.class.bt_passthrough_values
+      end
+
+      def key?(key)
+        if @context.class.bt_passthrough_values.include?(key.to_s)
+          @context.public_send(key.to_sym).present?
+        else
+          inner_hash.key?(key)
+        end
+      end
+
+      # def to_h
+      #   inner_hash.merge(@context.class.bt_passthrough_values.index_with {|k| @context.public_send(k.to_sym) })
+      # end
+
+      private
+
+      def inner_hash
+        @context.extra_options
+      end
+    end
+  end
+end

data/app/models/joblin/background_task/retention_policy.rb

@@ -0,0 +1,28 @@
+module Joblin
+  module BackgroundTask::RetentionPolicy
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def record_retention(policy = nil)
+        if policy.nil?
+          @record_retention || superclass.try(:record_retention)
+        else
+          @record_retention = policy
+        end
+      end
+    end
+
+    class BackgroundTaskCleaner < ActiveJob::Base
+      def perform
+        types = BackgroundTask.distinct.pluck(:type)
+        types.each do |type|
+          type = type.constantize
+          rp = type.record_retention
+          next unless rp
+
+          type.where("created_at < ?", rp.ago).destroy_all
+        end
+      end
+    end
+  end
+end

data/app/models/joblin/background_task.rb

@@ -0,0 +1,72 @@
+module Joblin
+  class BackgroundTask < ActiveRecord::Base
+    include Options
+    include Executor
+    include RetentionPolicy
+    include Attachments
+    include ApiAccess
+
+    belongs_to :creator, polymorphic: true, optional: true
+
+    after_initialize do
+      self.workflow_state ||= 'unscheduled'
+    end
+
+    scope :grouped_history, lambda { |history_length|
+      t = quoted_table_name
+      if history_length > 1
+        joins("LEFT OUTER JOIN #{t} AS t2 ON #{t}.type = t2.type AND #{t}.created_at < t2.created_at")
+          .group(:id).order(:type, created_at: :desc).having('count(*) < ?', history_length)
+      else
+        select("DISTINCT ON (#{t}.type) #{t}.*")
+      end
+    }
+
+    def enqueue!
+      self.workflow_state = 'scheduled' if self.workflow_state == 'unscheduled'
+
+      enter_batch do
+        self.class::ExecutorJob.perform_later()
+      end
+    end
+
+    def cancel!
+      update!(workflow_state: 'cancelled')
+    end
+
+    protected
+
+    def perform
+      raise NotImplementedError
+    end
+
+    def enter_batch(&blk)
+      b = batch_id.present? ? Joblin::Batching::Batch.new(batch_id) : Joblin::Batching::Batch.new.tap do |b|
+        update!(batch_id: b.bid)
+        b.description = "BackgroundTask #{id}"
+        b.on(:complete, BackgroundTaskCallbacks, id: id)
+        b.on(:success, BackgroundTaskCallbacks, id: id)
+        b.context[:background_task_id] = id
+      end
+      b.jobs(&blk)
+    end
+
+    class BackgroundTaskCallbacks
+      def on_complete(status, options)
+        tracker = BackgroundTask.find(options['id'])
+        return if tracker.workflow_state == 'cancelled'
+
+        tracker.workflow_state = status.success? ? 'completed' : 'failed'
+        tracker.save! if tracker.changed?
+      end
+
+      def on_success(status, options)
+        tracker = BackgroundTask.find(options['id'])
+        return if tracker.workflow_state == 'cancelled'
+
+        tracker.workflow_state = 'completed'
+        tracker.save! if tracker.changed?
+      end
+    end
+  end
+end

data/app/models/joblin/concerns/job_working_dirs.rb

@@ -0,0 +1,21 @@
+module Joblin::Concerns
+  module JobWorkingDirs
+    extend ActiveSupport::Concern
+
+    included do
+      around_perform :cleanup_working_dir
+    end
+
+    def working_dir
+      @working_dir ||= Dir.mktmpdir
+    end
+
+    private
+
+    def cleanup_working_dir
+      yield if block_given?
+    ensure
+      FileUtils.remove_entry @working_dir if @working_dir
+    end
+  end
+end

data/db/migrate/20250903184852_create_background_tasks.rb

@@ -0,0 +1,12 @@
+class CreateBackgroundTasks < ActiveRecord::Migration[5.2]
+  def change
+    create_table :joblin_background_tasks do |t|
+      t.references :creator, polymorphic: true
+      t.jsonb :extra_options
+      t.string :batch_id
+      t.string :type, null: false
+      t.string :workflow_state, null: false, default: 'unscheduled'
+      t.timestamps
+    end
+  end
+end

data/joblin.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+begin
+  require "joblin/version"
+  version = Joblin::VERSION
+rescue LoadError
+  version = "0.0.0.docker"
+end
+
+Gem::Specification.new do |spec|
+  spec.name          = "joblin"
+  spec.version       = version
+  spec.authors       = ["Ethan Knapp"]
+  spec.email         = ["eknapp@instructure.com"]
+
+  spec.summary       = "Gem for ActiveJob extensions"
+  spec.homepage      = "https://instructure.com"
+
+  spec.files = Dir["{app,config,db,lib}/**/*", "README.md", "*.gemspec"]
+  spec.test_files = Dir["spec/**/*"]
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'rediconn', '~> 0.1.0'
+  spec.add_dependency 'rails', '>= 5', '< 9.0'
+
+  spec.add_development_dependency 'rspec', '~> 3'
+  spec.add_development_dependency 'redis'
+  spec.add_development_dependency 'rspec-rails'
+  spec.add_development_dependency 'pg'
+
+  spec.add_development_dependency "appraisal"
+  spec.add_development_dependency 'combustion', '~> 1.3'
+end