marty 9.5.1 → 10.0.0

data/app/jobs/marty/delorean_background_job.rb

@@ -0,0 +1,8 @@
+module Marty
+  class DeloreanBackgroundJob < ::Marty::CronJob
+    def perform(script, node, attribute)
+      engine = Marty::ScriptSet.new.get_engine(script)
+      engine.evaluate(node, attribute, {})
+    end
+  end
+end

data/app/models/marty/background_job/schedule.rb

@@ -8,20 +8,43 @@ module Marty
       REGEX = %r{\A(((\*?[\d/,\-]*)\s){3,4}(\*?([\d/,\-])*\s)(\*?([\d/,\-])*))\z}i
 
       validates :job_class, :cron, :state, presence: true
-      validates :job_class, uniqueness: true
       validates :cron, format: { with: REGEX }
 
       validate :job_class_validation
+      validate :arguments_array_validation
+      validate :job_class_uniqueness_validation
+
+      has_one :delayed_job, class_name: '::Delayed::Job', dependent: :destroy
 
       ALL_STATES = %w[on off].freeze
       enum state: ALL_STATES.zip(ALL_STATES).to_h
 
+      scope :by_arguments, lambda { |arguments|
+        where('arguments = ?', arguments.to_json)
+      }
+
       def job_class_validation
         job_class.constantize.respond_to?(:schedule)
       rescue NameError
         errors.add(:job_class, "doesn't exist")
         false
       end
+
+      def arguments_array_validation
+        return if arguments.is_a? Array
+
+        errors.add(:arguments, 'must be an Array')
+        false
+      end
+
+      def job_class_uniqueness_validation
+        return unless self.class.by_arguments(arguments).
+                        where.not(id: id).
+                        where(job_class: job_class).any?
+
+        errors.add(:arguments, 'are not unique')
+        false
+      end
     end
   end
 end

data/app/models/marty/delorean_rule.rb

@@ -1,7 +1,7 @@
 class Marty::DeloreanRule < Marty::BaseRule
   self.abstract_class = true
 
-  validates :rule_type, :start_dt, presence: true
+  validates :start_dt, presence: true
 
   def validate
     super

data/app/models/marty/vw_promise.rb

@@ -23,16 +23,18 @@ class Marty::VwPromise < Marty::Base
     def user_id
       0
     end
-    alias_method :job_id, :user_id
 
-    def result
-      nil
+    def job_id
+      0
     end
-    [:start_dt, :end_dt].each { |m| alias_method m, :result }
 
     def status
       true
     end
+
+    [:result, :start_dt, :end_dt, :timeout].each do |m|
+      define_method(m) { nil }
+    end
   end
 
   def self.root

data/app/services/marty/background_job/fetch_missing_in_schedule_cron_jobs.rb

@@ -2,17 +2,19 @@ module Marty
   module BackgroundJob
     module FetchMissingInScheduleCronJobs
       def self.call
-        in_dashboard = Marty::BackgroundJob::Schedule.pluck(:job_class)
+        in_dashboard = Marty::BackgroundJob::Schedule.all
 
-        names_conditions = in_dashboard.map do |job_class_name|
-          "%job_class: #{job_class_name}\n%"
+        names_conditions = in_dashboard.map do |schedule|
+          "%job_class: #{schedule.job_class}\n%"
         end
 
-        Delayed::Job.
+        djs = Delayed::Job.
           where.not(cron: nil).
           where.not(cron: '').
-          where("handler ILIKE '%job_class:%'").
-          where.not('handler ILIKE ANY ( array[?] )', names_conditions)
+          where("handler ILIKE '%job_class:%'")
+
+        djs.where.not('handler ILIKE ANY ( array[?] )', names_conditions).
+          or(djs.where.not(schedule_id: in_dashboard.map(&:id)))
       end
     end
   end

data/app/services/marty/background_job/update_schedule.rb

@@ -4,26 +4,33 @@ module Marty
       def self.call(id:, job_class:)
         model = Marty::BackgroundJob::Schedule.find_by(id: id)
 
-        return remove_schedule(job_class: job_class) if model.blank?
-        return remove_schedule(job_class: job_class) if model.off?
-        return schedule(job_class: job_class) if model.on?
+        if model.blank? || model.off?
+          return remove_schedule(
+            schedule_id: id,
+            job_class: job_class
+          )
+        end
+
+        return schedule(schedule_obj: model) if model.on?
       end
 
-      def self.remove_schedule(job_class:)
+      def self.remove_schedule(schedule_id:, job_class:)
         klass = job_class.constantize
-        klass.remove_schedule if klass.respond_to?(:remove_schedule)
+        return true unless klass.respond_to?(:remove_schedule)
+
+        klass.remove_schedule(Delayed::Job.find_by(schedule_id: schedule_id))
 
         true
       rescue NameError
         false
       end
 
-      def self.schedule(job_class:)
-        klass = job_class.constantize
+      def self.schedule(schedule_obj:)
+        klass = schedule_obj.job_class.constantize
 
         return false unless klass.respond_to?(:schedule)
 
-        klass.schedule
+        klass.schedule(schedule_obj: schedule_obj)
 
         true
       rescue NameError

data/app/services/marty/jobs/schedule.rb

@@ -3,7 +3,7 @@ module Marty
     module Schedule
       extend Delorean::Functions
 
-      delorean_fn :call, sig: 0 do
+      delorean_fn :call do
         glob = Rails.root.join('app/jobs/**/*_job.rb')
         Dir.glob(glob).sort.each { |f| require f }
 
@@ -12,9 +12,13 @@ module Marty
 
         Delayed::Job.where.not(cron: nil).each(&:destroy!)
 
-        Marty::CronJob.subclasses.map do |klass|
-          klass.schedule
-          [klass.name, klass.cron_expression]
+        Marty::BackgroundJob::Schedule.all.map do |schedule|
+          Marty::BackgroundJob::UpdateSchedule.call(
+            id: schedule.id,
+            job_class: schedule.job_class,
+          )
+
+          [schedule.job_class, schedule.arguments, schedule.cron]
         end
       end
     end

data/app/services/marty/promises/delorean/create.rb

@@ -27,7 +27,8 @@ module Marty
             user_id: promise_params[:_user_id],
             parent_id: promise_params[:_parent_id],
             priority: priority,
-            promise_type: 'delorean'
+            promise_type: 'delorean',
+            timeout: timeout
           )
 
           params[:_promise_id] = promise.id

data/app/services/marty/promises/ruby/create.rb

@@ -27,7 +27,8 @@ module Marty
             user_id: promise_params[:_user_id],
             parent_id: promise_params[:_parent_id],
             priority: priority,
-            promise_type: 'ruby'
+            promise_type: 'ruby',
+            timeout: timeout
           )
 
           begin

data/config/locales/en.yml

@@ -134,6 +134,7 @@ en:
     title: Job Name
     start_dt: Start Date
     end_dt: End Date
+    total_time: Total Time
     user_login: User Login
     cformat: Format
     schedule_dashboard:

data/db/migrate/510_schedule_job_to_remove_old_promises.rb

@@ -2,13 +2,15 @@ class ScheduleJobToRemoveOldPromises < ActiveRecord::Migration[4.2]
   def up
     cron_every_hour = '0 * * * *'
 
-    Marty::BackgroundJob::Schedule.create!(
+    schedule = Marty::BackgroundJob::Schedule.new(
       job_class: 'Marty::RemoveOldPromisesJob',
       cron: cron_every_hour,
       state: 'on'
     )
 
-    ::Marty::RemoveOldPromisesJob.schedule
+    # Since we add `arguments` column to the model in later migrations,
+    # we should skip it's validation here
+    schedule.save!(validate: false)
   end
 
   def down

data/db/migrate/523_add_timeout_to_promises.rb

@@ -0,0 +1,5 @@
+class AddTimeoutToPromises < ActiveRecord::Migration[4.2]
+  def change
+    add_column :marty_promises, :timeout, :integer
+  end
+end

data/db/migrate/524_add_timeout_to_promise_view.rb

@@ -0,0 +1,45 @@
+class AddTimeoutToPromiseView < ActiveRecord::Migration[4.2]
+  def up
+    execute <<~SQL
+       DROP VIEW IF EXISTS marty_vw_promises;
+       CREATE OR REPLACE VIEW marty_vw_promises
+       AS
+       SELECT
+       id,
+       title,
+       user_id,
+       cformat,
+       parent_id,
+       job_id,
+       status,
+       start_dt,
+       end_dt,
+       priority,
+       timeout
+       FROM marty_promises;
+
+       GRANT SELECT ON marty_vw_promises TO public;
+    SQL
+  end
+
+  def down
+    execute <<~SQL
+       DROP VIEW IF EXISTS marty_vw_promises;
+       CREATE OR REPLACE VIEW marty_vw_promises
+       AS
+       SELECT
+       id,
+       title,
+       user_id,
+       cformat,
+       parent_id,
+       job_id,
+       status,
+       start_dt,
+       end_dt
+       FROM marty_promises;
+
+       GRANT SELECT ON marty_vw_promises TO public;
+    SQL
+  end
+end

data/db/migrate/525_add_arguments_to_jobs_schedules.rb

@@ -0,0 +1,10 @@
+class AddArgumentsToJobsSchedules < ActiveRecord::Migration[5.1]
+  def change
+    add_column :marty_background_job_schedules, :arguments, :jsonb, default: [], null: false
+
+    remove_index :marty_background_job_schedules, :job_class
+    add_index :marty_background_job_schedules, [:job_class, :arguments], unique: true
+
+    add_column :marty_background_job_logs, :arguments, :jsonb, default: [], null: false
+  end
+end

data/db/migrate/526_add_schedule_id_to_delayed_jobs.rb

@@ -0,0 +1,14 @@
+class AddScheduleIdToDelayedJobs < ActiveRecord::Migration[5.1]
+  def change
+    add_column :delayed_jobs, :schedule_id, :integer
+    add_index :delayed_jobs, :schedule_id
+
+    reversible do |dir|
+      dir.up { set_ids }
+    end
+  end
+
+  def set_ids
+    ::Marty::Jobs::Schedule.call
+  end
+end

data/lib/marty/delayed_job/queue_adapter.rb

@@ -0,0 +1,38 @@
+# Copied from https://github.com/codez/delayed_cron_job/blob/master/lib/delayed_cron_job/active_job/queue_adapter.rb
+# Only schedule_id is added
+module Marty
+  module DelayedJob
+    module QueueAdapter
+      def self.included(klass)
+        klass.send(:alias_method, :enqueue, :enqueue_with_cron)
+        klass.send(:alias_method, :enqueue_at, :enqueue_at_with_cron)
+      end
+
+      def self.extended(klass)
+        meta = class << klass; self; end
+        meta.send(:alias_method, :enqueue, :enqueue_with_cron)
+        meta.send(:alias_method, :enqueue_at, :enqueue_at_with_cron)
+      end
+
+      def enqueue_with_cron(job)
+        enqueue_at(job, nil)
+      end
+
+      def enqueue_at_with_cron(job, timestamp)
+        options = {
+          queue: job.queue_name,
+          cron: job.cron,
+          schedule_id: job.schedule_id
+        }
+
+        options[:run_at] = Time.at(timestamp) if timestamp
+        options[:priority] = job.priority if job.respond_to?(:priority)
+        wrapper = ::ActiveJob::QueueAdapters::DelayedJobAdapter::JobWrapper.new(job.serialize)
+        delayed_job = Delayed::Job.enqueue(wrapper, options)
+        job.provider_job_id = delayed_job.id if job.respond_to?(:provider_job_id=)
+
+        delayed_job
+      end
+    end
+  end
+end

data/lib/marty/delayed_job/scheduled_job_plugin.rb

@@ -14,17 +14,20 @@ module Marty
         lifecycle.before(:error) do |worker, job, &block|
           if cron?(job)
             begin
-              job_class_str = job.handler.split("\n").find do |line|
-                line.include? 'job_class'
+              schedule = ::Marty::BackgroundJob::Schedule.find_by(id: job.schedule_id)
+
+              if schedule&.on?
+                job.cron = schedule.cron
+                job.schedule_id = schedule.id
+              else
+                job.cron = nil
+                job.schedule_id = nil
               end
-              job_class_name = job_class_str.gsub('job_class:', '').strip
-              job_class = job_class_name.constantize
-              job.cron = job_class.cron_expression
             rescue StandardError
             end
           else
             # No cron job - proceed as normal
-            block.call(worker, job)
+            block&.call(worker, job)
           end
         end
       end

data/lib/marty/diagnostic/environment_variables.rb

@@ -4,7 +4,7 @@ module Marty::Diagnostic; class EnvironmentVariables < Base
   end
 
   def self.env filter = ''
-    env = ENV.clone
+    env = ENV.to_h.clone
 
     to_delete = (Marty::Config['DIAG_ENV_BLOCK'] || []).map(&:upcase) + [
       'SCRIPT_URI', 'SCRIPT_URL'

data/lib/marty/monkey.rb

@@ -396,5 +396,16 @@ end
 
 require 'delayed_cron_job'
 require_relative './delayed_job/scheduled_job_plugin.rb'
+require_relative './delayed_job/queue_adapter.rb'
 
 Delayed::Worker.plugins << Marty::DelayedJob::ScheduledJobPlugin
+
+if ActiveJob::QueueAdapters::DelayedJobAdapter.respond_to?(:enqueue)
+  ActiveJob::QueueAdapters::DelayedJobAdapter.extend(
+    Marty::DelayedJob::QueueAdapter
+  )
+else
+  ActiveJob::QueueAdapters::DelayedJobAdapter.include(
+    Marty::DelayedJob::QueueAdapter
+  )
+end

data/lib/marty/promise_job.rb

@@ -43,8 +43,9 @@ class Marty::PromiseJob < Struct.new(:promise,
 
       # log "DONE #{Process.pid} #{promise.id} #{Time.now.to_f} #{res}"
     rescue ::Delayed::WorkerTimeout => e
+      msg = ::Marty::Promise.timeout_message(promise)
       timeout_error = StandardError.new(
-        ::Marty::Promise.timeout_message(promise)
+        "#{msg} (Triggered by #{e.class})"
       )
       timeout_error.set_backtrace(e.backtrace)
 

data/lib/marty/promise_ruby_job.rb

@@ -28,8 +28,9 @@ class Marty::PromiseRubyJob < Struct.new(:promise,
       mod = module_name.constantize
       res = { 'result' => mod.send(method_name, *method_args) }
     rescue ::Delayed::WorkerTimeout => e
+      msg = ::Marty::Promise.timeout_message(promise)
       timeout_error = StandardError.new(
-        ::Marty::Promise.timeout_message(promise)
+        "#{msg} (Triggered by #{e.class})"
       )
       timeout_error.set_backtrace(e.backtrace)
 

data/lib/marty/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Marty
-  VERSION = '9.5.1'
+  VERSION = '10.0.0'
 end

data/spec/dummy/.foreman

@@ -0,0 +1,2 @@
+formation: worker=4,web=1
+port: 3000

data/spec/dummy/Procfile

@@ -0,0 +1,2 @@
+web: bundle exec rails s -e ${RAILS_ENV:-development}
+worker: bundle exec rake jobs:work