canvas_sync 0.16.5 → 0.17.0.beta5

data/spec/job_batching/status_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::JobBatches::Batch::Status do
+  let(:bid) { 'BID' }
+  let(:batch) { CanvasSync::JobBatches::Batch.new(bid) }
+  subject { described_class.new(bid) }
+
+  describe '#join' do
+    it 'raises info' do
+      expect { subject.join }.to raise_error('Not supported')
+    end
+  end
+
+  describe '#pending' do
+    context 'when not initalized' do
+      it 'returns 0 pending jobs' do
+        expect(subject.pending).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.jobs do TestWorker.perform_async end }
+      it 'returns pending jobs' do
+        expect(subject.pending).to eq(1)
+      end
+    end
+  end
+
+  describe '#failures' do
+    context 'when not initalized' do
+      it 'returns 0 failed jobs' do
+        expect(subject.failures).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.increment_job_queue(bid) }
+      before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'FAILEDID') }
+
+      it 'returns failed jobs' do
+        expect(subject.failures).to eq(1)
+      end
+    end
+  end
+
+  describe '#failure_info' do
+    context 'when not initalized' do
+      it 'returns empty array' do
+        expect(subject.failure_info).to eq([])
+      end
+    end
+
+    context 'when with error' do
+      before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'jid123') }
+
+      it 'returns array with failed jids' do
+        expect(subject.failure_info).to eq(['jid123'])
+      end
+    end
+  end
+
+  describe '#total' do
+    context 'when not initalized' do
+      it 'returns 0 failed jobs' do
+        expect(subject.total).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.jobs do TestWorker.perform_async end }
+
+      it 'returns failed jobs' do
+        expect(subject.total).to eq(1)
+      end
+    end
+  end
+
+  describe '#data' do
+    it 'returns batch description' do
+      expect(subject.data).to include(total: 0, failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil)
+    end
+  end
+
+  describe '#created_at' do
+    it 'returns time' do
+      batch = CanvasSync::JobBatches::Batch.new
+      batch.jobs do TestWorker.perform_async end
+      status = described_class.new(batch.bid)
+      expect(status.created_at).not_to be_nil
+    end
+  end
+end

data/spec/job_batching/support/base_job.rb

@@ -0,0 +1,14 @@
+class BatchTestJobBase < ActiveJob::Base
+  def perform
+  end
+
+  def self.perform_async(*args)
+    perform_later(*args)
+  end
+end
+
+class FailingBatchTestJobBase < BatchTestJobBase
+  def perform
+    raise "Foo"
+  end
+end

data/spec/job_batching/support/sample_callback.rb

@@ -0,0 +1,2 @@
+  class SampleCallback; end
+  class SampleCallback2; end

data/spec/spec_helper.rb

@@ -13,6 +13,19 @@ require 'shoulda/matchers'
 require 'pry'
 require 'pry-nav'
 
+require 'sidekiq/testing'
+Sidekiq::Testing.fake!
+
+require 'fakeredis/rspec'
+Dir[File.dirname(__FILE__) + "/job_batching/support/**/*.rb"].each {|f| require f }
+
+# Fix an issue with fakeredis with Redis >=4.2
+class Redis::Connection::Memory
+  def exists(*keys)
+    keys.count { |key| data.key?(key) }
+  end
+end
+
 ActiveRecord::Migration.maintain_test_schema!
 
 RSpec.configure do |config|
@@ -51,3 +64,7 @@ end
 def canvas_sync_client
   Bearcat::Client.new(token: 'cool-token', prefix: 'http://test.instructure.com')
 end
+
+def set_batch_context(ctx)
+  allow_any_instance_of(ActiveJob::Base).to receive(:batch_context).and_return(ctx.with_indifferent_access)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.16.5
+  version: 0.17.0.beta5
 platform: ruby
 authors:
 - Nate Collings
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-27 00:00:00.000000000 Z
+date: 2020-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -234,6 +234,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activejob
   requirement: !ruby/object:Gem::Requirement
@@ -248,6 +276,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -316,10 +358,12 @@ files:
 - app/controllers/api/v1/health_check_controller.rb
 - app/controllers/api/v1/live_events_controller.rb
 - app/models/canvas_sync/job_log.rb
+- app/models/canvas_sync/sync_batch.rb
 - config/initializers/apartment.rb
 - db/migrate/20170915210836_create_canvas_sync_job_log.rb
 - db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb
 - db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb
+- db/migrate/20201018210836_create_canvas_sync_sync_batches.rb
 - lib/canvas_sync.rb
 - lib/canvas_sync/api_syncable.rb
 - lib/canvas_sync/class_callback_executor.rb
@@ -377,8 +421,17 @@ files:
 - lib/canvas_sync/importers/bulk_importer.rb
 - lib/canvas_sync/importers/legacy_importer.rb
 - lib/canvas_sync/job.rb
-- lib/canvas_sync/job_chain.rb
-- lib/canvas_sync/jobs/fork_gather.rb
+- lib/canvas_sync/job_batches/batch.rb
+- lib/canvas_sync/job_batches/batch_aware_job.rb
+- lib/canvas_sync/job_batches/callback.rb
+- lib/canvas_sync/job_batches/chain_builder.rb
+- lib/canvas_sync/job_batches/context_hash.rb
+- lib/canvas_sync/job_batches/jobs/base_job.rb
+- lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb
+- lib/canvas_sync/job_batches/jobs/serial_batch_job.rb
+- lib/canvas_sync/job_batches/sidekiq.rb
+- lib/canvas_sync/job_batches/status.rb
+- lib/canvas_sync/jobs/begin_sync_chain_job.rb
 - lib/canvas_sync/jobs/report_checker.rb
 - lib/canvas_sync/jobs/report_processor_job.rb
 - lib/canvas_sync/jobs/report_starter.rb
@@ -407,7 +460,6 @@ files:
 - lib/canvas_sync/sidekiq_job.rb
 - lib/canvas_sync/version.rb
 - spec/canvas_sync/canvas_sync_spec.rb
-- spec/canvas_sync/jobs/fork_gather_spec.rb
 - spec/canvas_sync/jobs/job_spec.rb
 - spec/canvas_sync/jobs/report_checker_spec.rb
 - spec/canvas_sync/jobs/report_processor_job_spec.rb
@@ -542,6 +594,19 @@ files:
 - spec/factories/submission_factory.rb
 - spec/factories/term_factory.rb
 - spec/factories/user_factory.rb
+- spec/job_batching/batch_aware_job_spec.rb
+- spec/job_batching/batch_spec.rb
+- spec/job_batching/callback_spec.rb
+- spec/job_batching/flow_spec.rb
+- spec/job_batching/integration/integration.rb
+- spec/job_batching/integration/nested.rb
+- spec/job_batching/integration/simple.rb
+- spec/job_batching/integration/workflow.rb
+- spec/job_batching/integration_helper.rb
+- spec/job_batching/sidekiq_spec.rb
+- spec/job_batching/status_spec.rb
+- spec/job_batching/support/base_job.rb
+- spec/job_batching/support/sample_callback.rb
 - spec/spec_helper.rb
 - spec/support/fake_canvas.rb
 - spec/support/fixtures/canvas_responses/admins.json
@@ -578,9 +643,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.1.2
 signing_key:
@@ -588,7 +653,6 @@ specification_version: 4
 summary: Gem for generating Canvas models and migrations and syncing data from Canvas
 test_files:
 - spec/canvas_sync/canvas_sync_spec.rb
-- spec/canvas_sync/jobs/fork_gather_spec.rb
 - spec/canvas_sync/jobs/job_spec.rb
 - spec/canvas_sync/jobs/report_checker_spec.rb
 - spec/canvas_sync/jobs/report_processor_job_spec.rb
@@ -723,6 +787,19 @@ test_files:
 - spec/factories/submission_factory.rb
 - spec/factories/term_factory.rb
 - spec/factories/user_factory.rb
+- spec/job_batching/batch_aware_job_spec.rb
+- spec/job_batching/batch_spec.rb
+- spec/job_batching/callback_spec.rb
+- spec/job_batching/flow_spec.rb
+- spec/job_batching/integration/integration.rb
+- spec/job_batching/integration/nested.rb
+- spec/job_batching/integration/simple.rb
+- spec/job_batching/integration/workflow.rb
+- spec/job_batching/integration_helper.rb
+- spec/job_batching/sidekiq_spec.rb
+- spec/job_batching/status_spec.rb
+- spec/job_batching/support/base_job.rb
+- spec/job_batching/support/sample_callback.rb
 - spec/spec_helper.rb
 - spec/support/fake_canvas.rb
 - spec/support/fixtures/canvas_responses/admins.json

data/lib/canvas_sync/job_chain.rb

@@ -1,102 +0,0 @@
-module CanvasSync
-  class JobChain
-    attr_reader :chain_data
-
-    delegate_missing_to :chain_data
-
-    VALID_PLACEMENT_PARAMETERS = %i[before after].freeze
-
-    def initialize(chain_data)
-      @chain_data = chain_data
-    end
-
-    def jobs
-      chain_data[:jobs]
-    end
-
-    def global_options
-      chain_data[:global_options]
-    end
-
-    def merge_options(job, options)
-      jobs.each do |j|
-        j[:options].deep_merge!(options) if job_matches_pattern(j, job)
-      end
-    end
-
-    def insert(new_job, **kwargs)
-      invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
-      raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
-      raise "Exactly one placement parameter may be provided" if kwargs.values.compact!.length > 1
-
-      if !kwargs.present?
-        jobs << new_job
-      else
-        placement = kwargs.keys[0]
-        relative_to = kwargs.values[0]
-
-        index = jobs.index { |job| job_matches_pattern(job, relative_to) }
-        raise "Could not find a \"#{relative_to}\" job in the chain" if index.nil?
-
-        index += 1 if placement == :after
-        new_job[:job] = new_job[:job].to_s
-        jobs.insert(index, new_job)
-      end
-    end
-
-    def process!(extra_options: {})
-      perform_next(extra_options)
-    end
-
-    def duplicate
-      self.class.new(Marshal.load(Marshal.dump(chain_data)))
-    end
-
-    def normalize!
-      @chain_data[:global_options] ||= {}
-    end
-
-    def serialize
-      normalize!
-      chain_data
-    end
-
-    def perform_next(extra_options = {})
-      return if jobs.empty?
-
-      # Make sure all job classes are serialized as strings
-      jobs.each { |job| job[:job] = job[:job].to_s }
-
-      duped_job_chain = duplicate
-
-      jobs = duped_job_chain[:jobs]
-      next_job = jobs.shift
-      next_job_class = next_job[:job].constantize
-      next_options = next_job[:options] || {}
-      next_options.merge!(extra_options)
-      next_job_class.perform_later(duped_job_chain.serialize, next_options)
-    end
-
-    def fork(job_log, keys: [])
-      duped_job_chain = duplicate
-      duped_job_chain[:fork_state] ||= {}
-      duped_job_chain[:fork_state][:forking_path] ||= []
-      duped_job_chain[:fork_state][:pre_fork_globals] ||= []
-
-      duped_job_chain[:fork_state][:forking_path] << job_log.job_id
-      duped_job_chain[:fork_state][:pre_fork_globals] << global_options
-      # duped_job_chain[:global_options][:on_failure] ||= ['CanvasSync::Jobs::ForkGather.handle_branch_error']
-
-      sub_items = yield duped_job_chain
-      sub_count = sub_items.respond_to?(:count) ? sub_items.count : sub_items
-      job_log.update!(fork_count: sub_count)
-      sub_items
-    end
-
-    private
-
-    def job_matches_pattern(job_entry, pattern)
-      job_entry[:job].to_s == pattern.to_s
-    end
-  end
-end

data/lib/canvas_sync/jobs/fork_gather.rb

@@ -1,74 +0,0 @@
-module CanvasSync
-  module Jobs
-    class ForkGather < CanvasSync::Job
-      def perform(job_chain, options)
-        forked_job = self.class.forked_at_job(job_chain)
-
-        while true
-          if forked_job.present?
-            forked_job.with_lock do
-              forked_job.fork_count -= 1
-              forked_job.save!
-            end
-
-            if forked_job.fork_count <= 0
-              pfgs = job_chain[:fork_state][:pre_fork_globals].pop
-              job_chain[:global_options] = pfgs
-
-              if options[:gather_all]
-                # If we want to gather all, repeat for the next level fork
-                forked_job = self.class.forked_at_job(job_chain)
-              else
-                forked_job = nil
-              end
-            else
-              # If a fork was found and it isn't complete, break the loop before continuing the chain
-              break
-            end
-
-            # Repeat this logic for [if gather_all] the next fork up, or [if not gather_all] nil
-            next
-          end
-
-          # If there is no current fork (either not in a fork, or all forks were closed), continue the chain
-          CanvasSync.invoke_next(job_chain)
-          break
-        end
-      end
-
-      def self.handle_branch_error(e, job_chain:, skip_invoke: false, **kwargs)
-        return nil unless job_chain&.dig(:fork_state, :forking_path).present?
-
-        duped_chain = CanvasSync.duplicate_chain(job_chain)
-        job_list = duped_chain[:jobs]
-        while job_list.count > 0
-          job_class_name = job_list[0][:job]
-          job_class = job_class_name.constantize
-          break if job_class <= CanvasSync::Jobs::ForkGather
-          job_list.shift
-        end
-
-        return nil unless job_list.present?
-
-        if skip_invoke
-          duped_chain
-        else
-          CanvasSync.invoke_next(duped_chain)
-          true
-        end
-      end
-
-      protected
-
-      def self.forked_at_job(job_chain)
-        fork_item = (job_chain.dig(:fork_state, :forking_path) || []).pop
-
-        if fork_item.present?
-          CanvasSync::JobLog.find_by(job_id: fork_item)
-        else
-          nil
-        end
-      end
-    end
-  end
-end