canvas_sync 0.16.4 → 0.17.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +49 -137
- data/app/models/canvas_sync/sync_batch.rb +5 -0
- data/db/migrate/20170915210836_create_canvas_sync_job_log.rb +12 -31
- data/db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb +4 -13
- data/db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb +3 -11
- data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb +11 -0
- data/lib/canvas_sync.rb +35 -118
- data/lib/canvas_sync/job.rb +5 -5
- data/lib/canvas_sync/job_batches/batch.rb +399 -0
- data/lib/canvas_sync/job_batches/batch_aware_job.rb +62 -0
- data/lib/canvas_sync/job_batches/callback.rb +153 -0
- data/lib/canvas_sync/job_batches/chain_builder.rb +203 -0
- data/lib/canvas_sync/job_batches/context_hash.rb +147 -0
- data/lib/canvas_sync/job_batches/jobs/base_job.rb +7 -0
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +18 -0
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +73 -0
- data/lib/canvas_sync/job_batches/sidekiq.rb +91 -0
- data/lib/canvas_sync/job_batches/status.rb +63 -0
- data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +34 -0
- data/lib/canvas_sync/jobs/report_checker.rb +3 -6
- data/lib/canvas_sync/jobs/report_processor_job.rb +2 -5
- data/lib/canvas_sync/jobs/report_starter.rb +27 -19
- data/lib/canvas_sync/jobs/sync_accounts_job.rb +3 -5
- data/lib/canvas_sync/jobs/sync_admins_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_assignments_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_context_module_items_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_context_modules_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +4 -31
- data/lib/canvas_sync/jobs/sync_roles_job.rb +2 -5
- data/lib/canvas_sync/jobs/sync_simple_table_job.rb +11 -32
- data/lib/canvas_sync/jobs/sync_submissions_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_terms_job.rb +22 -7
- data/lib/canvas_sync/misc_helper.rb +15 -0
- data/lib/canvas_sync/version.rb +1 -1
- data/spec/canvas_sync/canvas_sync_spec.rb +126 -153
- data/spec/canvas_sync/jobs/job_spec.rb +9 -17
- data/spec/canvas_sync/jobs/report_checker_spec.rb +1 -3
- data/spec/canvas_sync/jobs/report_processor_job_spec.rb +0 -3
- data/spec/canvas_sync/jobs/report_starter_spec.rb +19 -28
- data/spec/canvas_sync/jobs/sync_admins_job_spec.rb +1 -4
- data/spec/canvas_sync/jobs/sync_assignment_groups_job_spec.rb +2 -1
- data/spec/canvas_sync/jobs/sync_assignments_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_context_module_items_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_context_modules_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +3 -35
- data/spec/canvas_sync/jobs/sync_roles_job_spec.rb +1 -4
- data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +5 -12
- data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb +2 -1
- data/spec/canvas_sync/jobs/sync_terms_job_spec.rb +1 -4
- data/spec/dummy/app/models/account.rb +3 -0
- data/spec/dummy/app/models/pseudonym.rb +14 -0
- data/spec/dummy/app/models/submission.rb +1 -0
- data/spec/dummy/app/models/user.rb +1 -0
- data/spec/dummy/config/environments/test.rb +2 -0
- data/spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb +24 -0
- data/spec/dummy/db/schema.rb +24 -4
- data/spec/job_batching/batch_aware_job_spec.rb +100 -0
- data/spec/job_batching/batch_spec.rb +363 -0
- data/spec/job_batching/callback_spec.rb +38 -0
- data/spec/job_batching/flow_spec.rb +91 -0
- data/spec/job_batching/integration/integration.rb +57 -0
- data/spec/job_batching/integration/nested.rb +88 -0
- data/spec/job_batching/integration/simple.rb +47 -0
- data/spec/job_batching/integration/workflow.rb +134 -0
- data/spec/job_batching/integration_helper.rb +48 -0
- data/spec/job_batching/sidekiq_spec.rb +124 -0
- data/spec/job_batching/status_spec.rb +92 -0
- data/spec/job_batching/support/base_job.rb +14 -0
- data/spec/job_batching/support/sample_callback.rb +2 -0
- data/spec/spec_helper.rb +10 -0
- metadata +90 -8
- data/lib/canvas_sync/job_chain.rb +0 -57
- data/lib/canvas_sync/jobs/fork_gather.rb +0 -59
- data/spec/canvas_sync/jobs/fork_gather_spec.rb +0 -73
| @@ -1,57 +0,0 @@ | |
| 1 | 
            -
            module CanvasSync
         | 
| 2 | 
            -
              class JobChain
         | 
| 3 | 
            -
                attr_reader :chain_data
         | 
| 4 | 
            -
             | 
| 5 | 
            -
                delegate_missing_to :chain_data
         | 
| 6 | 
            -
             | 
| 7 | 
            -
                VALID_PLACEMENT_PARAMETERS = %i[before after].freeze
         | 
| 8 | 
            -
             | 
| 9 | 
            -
                def initialize(chain_data)
         | 
| 10 | 
            -
                  @chain_data = chain_data
         | 
| 11 | 
            -
                end
         | 
| 12 | 
            -
             | 
| 13 | 
            -
                def jobs
         | 
| 14 | 
            -
                  chain_data[:jobs]
         | 
| 15 | 
            -
                end
         | 
| 16 | 
            -
             | 
| 17 | 
            -
                def global_options
         | 
| 18 | 
            -
                  chain_data[:global_options]
         | 
| 19 | 
            -
                end
         | 
| 20 | 
            -
             | 
| 21 | 
            -
                def merge_options(job, options)
         | 
| 22 | 
            -
                  jobs.each do |j|
         | 
| 23 | 
            -
                    j[:options].deep_merge!(options) if job_matches_pattern(j, job)
         | 
| 24 | 
            -
                  end
         | 
| 25 | 
            -
                end
         | 
| 26 | 
            -
             | 
| 27 | 
            -
                def insert(new_job, **kwargs)
         | 
| 28 | 
            -
                  invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
         | 
| 29 | 
            -
                  raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
         | 
| 30 | 
            -
                  raise "Exactly one placement parameter may be provided" if kwargs.values.compact!.length > 1
         | 
| 31 | 
            -
             | 
| 32 | 
            -
                  if !kwargs.present?
         | 
| 33 | 
            -
                    jobs << new_job
         | 
| 34 | 
            -
                  else
         | 
| 35 | 
            -
                    placement = kwargs.keys[0]
         | 
| 36 | 
            -
                    relative_to = kwargs.values[0]
         | 
| 37 | 
            -
             | 
| 38 | 
            -
                    index = jobs.index { |job| job_matches_pattern(job, relative_to) }
         | 
| 39 | 
            -
                    raise "Could not find a \"#{relative_to}\" job in the chain" if index.nil?
         | 
| 40 | 
            -
             | 
| 41 | 
            -
                    index += 1 if placement == :after
         | 
| 42 | 
            -
                    new_job[:job] = new_job[:job].to_s
         | 
| 43 | 
            -
                    jobs.insert(index, new_job)
         | 
| 44 | 
            -
                  end
         | 
| 45 | 
            -
                end
         | 
| 46 | 
            -
             | 
| 47 | 
            -
                def process!(extra_options: {})
         | 
| 48 | 
            -
                  CanvasSync::invoke_next(self, extra_options: extra_options)
         | 
| 49 | 
            -
                end
         | 
| 50 | 
            -
             | 
| 51 | 
            -
                private
         | 
| 52 | 
            -
             | 
| 53 | 
            -
                def job_matches_pattern(job_entry, pattern)
         | 
| 54 | 
            -
                  job_entry[:job].to_s == pattern.to_s
         | 
| 55 | 
            -
                end
         | 
| 56 | 
            -
              end
         | 
| 57 | 
            -
            end
         | 
| @@ -1,59 +0,0 @@ | |
| 1 | 
            -
            module CanvasSync
         | 
| 2 | 
            -
              module Jobs
         | 
| 3 | 
            -
                class ForkGather < CanvasSync::Job
         | 
| 4 | 
            -
                  def perform(job_chain, options)
         | 
| 5 | 
            -
                    forked_job = self.class.forked_at_job(job_chain)
         | 
| 6 | 
            -
             | 
| 7 | 
            -
                    if forked_job.present?
         | 
| 8 | 
            -
                      forked_job.with_lock do
         | 
| 9 | 
            -
                        forked_job.fork_count -= 1
         | 
| 10 | 
            -
                        forked_job.save!
         | 
| 11 | 
            -
                      end
         | 
| 12 | 
            -
             | 
| 13 | 
            -
                      if forked_job.fork_count <= 0
         | 
| 14 | 
            -
                        (job_chain[:global_options][:fork_keys] || []).pop&.each do |k|
         | 
| 15 | 
            -
                          job_chain[:global_options].delete(k.to_sym)
         | 
| 16 | 
            -
                        end
         | 
| 17 | 
            -
                        CanvasSync.invoke_next(job_chain)
         | 
| 18 | 
            -
                      end
         | 
| 19 | 
            -
                    else
         | 
| 20 | 
            -
                      CanvasSync.invoke_next(job_chain)
         | 
| 21 | 
            -
                    end
         | 
| 22 | 
            -
                  end
         | 
| 23 | 
            -
             | 
| 24 | 
            -
                  def self.handle_branch_error(e, job_chain:, skip_invoke: false, **kwargs)
         | 
| 25 | 
            -
                    return nil unless job_chain&.[](:global_options)&.[](:fork_path).present?
         | 
| 26 | 
            -
             | 
| 27 | 
            -
                    duped_chain = CanvasSync.duplicate_chain(job_chain)
         | 
| 28 | 
            -
                    job_list = duped_chain[:jobs]
         | 
| 29 | 
            -
                    while job_list.count > 0
         | 
| 30 | 
            -
                      job_class_name = job_list[0][:job]
         | 
| 31 | 
            -
                      job_class = job_class_name.constantize
         | 
| 32 | 
            -
                      break if job_class <= CanvasSync::Jobs::ForkGather
         | 
| 33 | 
            -
                      job_list.shift
         | 
| 34 | 
            -
                    end
         | 
| 35 | 
            -
             | 
| 36 | 
            -
                    return nil unless job_list.present?
         | 
| 37 | 
            -
             | 
| 38 | 
            -
                    if skip_invoke
         | 
| 39 | 
            -
                      duped_chain
         | 
| 40 | 
            -
                    else
         | 
| 41 | 
            -
                      CanvasSync.invoke_next(duped_chain)
         | 
| 42 | 
            -
                      true
         | 
| 43 | 
            -
                    end
         | 
| 44 | 
            -
                  end
         | 
| 45 | 
            -
             | 
| 46 | 
            -
                  protected
         | 
| 47 | 
            -
             | 
| 48 | 
            -
                  def self.forked_at_job(job_chain)
         | 
| 49 | 
            -
                    fork_item = (job_chain[:global_options][:fork_path] || []).pop
         | 
| 50 | 
            -
             | 
| 51 | 
            -
                    if fork_item.present?
         | 
| 52 | 
            -
                      CanvasSync::JobLog.find_by(job_id: fork_item)
         | 
| 53 | 
            -
                    else
         | 
| 54 | 
            -
                      nil
         | 
| 55 | 
            -
                    end
         | 
| 56 | 
            -
                  end
         | 
| 57 | 
            -
                end
         | 
| 58 | 
            -
              end
         | 
| 59 | 
            -
            end
         | 
| @@ -1,73 +0,0 @@ | |
| 1 | 
            -
            require 'spec_helper'
         | 
| 2 | 
            -
             | 
| 3 | 
            -
            RSpec.describe CanvasSync::Jobs::ForkGather do
         | 
| 4 | 
            -
              describe '#perform' do
         | 
| 5 | 
            -
                let!(:job_log) { CanvasSync::JobLog.create!(job_id: 'BLAH', fork_count: 3) }
         | 
| 6 | 
            -
                let(:job_chain) { {jobs: [], global_options: { fork_path: ['BLAH'] }} }
         | 
| 7 | 
            -
             | 
| 8 | 
            -
                it 'decrements fork_count' do
         | 
| 9 | 
            -
                  CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
         | 
| 10 | 
            -
                  expect(job_log.reload.fork_count).to eq 2
         | 
| 11 | 
            -
                end
         | 
| 12 | 
            -
             | 
| 13 | 
            -
                it 'does not continue chain if fork_count > 0' do
         | 
| 14 | 
            -
                  expect(CanvasSync).not_to receive(:invoke_next)
         | 
| 15 | 
            -
                  CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
         | 
| 16 | 
            -
                end
         | 
| 17 | 
            -
             | 
| 18 | 
            -
                it 'continues the chain if fork_count <= 0' do
         | 
| 19 | 
            -
                  job_log.update!(fork_count: 1)
         | 
| 20 | 
            -
                  expect(CanvasSync).to receive(:invoke_next)
         | 
| 21 | 
            -
                  CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
         | 
| 22 | 
            -
                end
         | 
| 23 | 
            -
             | 
| 24 | 
            -
                it 'continues if no fork_path is specified' do
         | 
| 25 | 
            -
                  job_chain[:global_options].delete(:fork_path)
         | 
| 26 | 
            -
                  expect(CanvasSync).to receive(:invoke_next)
         | 
| 27 | 
            -
                  CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
         | 
| 28 | 
            -
                end
         | 
| 29 | 
            -
             | 
| 30 | 
            -
                it 'pops the most-recent fork_path enrty' do
         | 
| 31 | 
            -
                  job_log.update!(fork_count: 1)
         | 
| 32 | 
            -
                  expect(CanvasSync).to receive(:invoke_next) do |*args|
         | 
| 33 | 
            -
                    expect(args[0][:global_options][:fork_path]).to eq []
         | 
| 34 | 
            -
                  end
         | 
| 35 | 
            -
                  CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
         | 
| 36 | 
            -
                end
         | 
| 37 | 
            -
              end
         | 
| 38 | 
            -
             | 
| 39 | 
            -
              describe 'handle_branch_error' do
         | 
| 40 | 
            -
                let(:error) { StandardError.new }
         | 
| 41 | 
            -
             | 
| 42 | 
            -
                let(:job_chain) {
         | 
| 43 | 
            -
                  {
         | 
| 44 | 
            -
                    jobs: [
         | 
| 45 | 
            -
                      { job: 'CanvasSync::Jobs::ReportChecker' },
         | 
| 46 | 
            -
                      { job: 'CanvasSync::Jobs::ReportChecker' },
         | 
| 47 | 
            -
                      { job: 'CanvasSync::Jobs::ForkGather' },
         | 
| 48 | 
            -
                      { job: 'CanvasSync::Jobs::ReportChecker' },
         | 
| 49 | 
            -
                    ],
         | 
| 50 | 
            -
                    global_options: {
         | 
| 51 | 
            -
                      fork_path: ['BLAH'],
         | 
| 52 | 
            -
                    }
         | 
| 53 | 
            -
                  }
         | 
| 54 | 
            -
                }
         | 
| 55 | 
            -
             | 
| 56 | 
            -
                it 'skips to and performs the next ForkGatherJob' do
         | 
| 57 | 
            -
                  expect(CanvasSync).to receive(:invoke_next) do |*args|
         | 
| 58 | 
            -
                    expect(args[0][:jobs][0][:job]).to eq 'CanvasSync::Jobs::ForkGather'
         | 
| 59 | 
            -
                  end
         | 
| 60 | 
            -
                  expect(CanvasSync::Jobs::ForkGather.handle_branch_error(error, job_chain: job_chain)).to be true
         | 
| 61 | 
            -
                end
         | 
| 62 | 
            -
             | 
| 63 | 
            -
                it 'does nothing if no ForkGather is in the chain' do
         | 
| 64 | 
            -
                  job_chain[:jobs].delete_at(2)
         | 
| 65 | 
            -
                  expect(CanvasSync::Jobs::ForkGather.handle_branch_error(error, job_chain: job_chain)).to be nil
         | 
| 66 | 
            -
                end
         | 
| 67 | 
            -
             | 
| 68 | 
            -
                it 'does nothing if no fork_path is present' do
         | 
| 69 | 
            -
                  job_chain[:global_options][:fork_path] = []
         | 
| 70 | 
            -
                  expect(CanvasSync::Jobs::ForkGather.handle_branch_error(error, job_chain: job_chain)).to be nil
         | 
| 71 | 
            -
                end
         | 
| 72 | 
            -
              end
         | 
| 73 | 
            -
            end
         |