canvas_sync 0.10.4 → 0.10.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3094e5606d612a10ba9f2fb9facf242d4fcfdbe561029f801642a5e1223b000b
-  data.tar.gz: ace716fc5b7e53a3d0f95849c17c1655dc130554ad33231db65099f41c283afa
+  metadata.gz: f1cc1054e379b920c5acee75583700fdffaae885a4cea48df1d8de97ed70c02e
+  data.tar.gz: 83741af3ec47ee4b72db5e4a354a8713c002fc6930cfe352a80c5f57a36dba4a
 SHA512:
-  metadata.gz: 2e94c64b6d9768069a03af99281c771720ebd411bb8d858a9bed9cc13fcf45d3f223e82b76c4018a42467c420131aac18cf4225ddf4c7eae37b55fffbea68d58
-  data.tar.gz: aa51f5e60f81e4b4fcf4808cb446947cc154a1f7d31df525621b792610acdeaec59edad52a26758ec47ece97e5c81e99bf66104157ae56a372fabee5bbbdd73d
+  metadata.gz: dc7ad8ad007bda5edb7cf980b767113075a4be84cf1c2a228cf8287f8aba2c587bb170574a0168216b0bcaa19c8c395224d37249954b23203a50df234e5d9599
+  data.tar.gz: be307cba8401af1b7697d4c843d260e290517aca0f3d3866a72db6f327f6d3726df894a5306f2b39f2e10eacbb22465a45b5b8398c810304d53145ef59a17ad7

data/README.md

@@ -91,6 +91,29 @@ This gem also helps with syncing and processing other reports if needed. In orde
 - Integrate your reports with the `ReportStarter`
 - Tell the gem what jobs to run
 
+### Extensible chain
+It is sometimes desired to extend or customize the chain of jobs that are run with CanvasSync.
+This can be achieved with the following pattern:
+
+```ruby
+job_chain = CanvasSync.default_provisioning_report_chain(
+  %i[list of models to sync]
+)
+
+# CanvasSync forks the chain for each term within a term scope. The ForkGather Job can be used to unfork the chain.
+# For example multiple Terms are in the term_scope. CanvasSync syncs Accounts, Terms, and Users (if enabled) and then
+# forks the chain to sync other models (eg Course, Sections, etc.) per-Term.
+# ForkGather will wait until all the forked chains are complete before continuing.
+# TL;DR: Jobs placed after SyncProvisioningReportJob and before ForkGather will run once per Term per Sync;
+# Jobs placed before SyncProvisioningReportJob or after ForkGather will run once per Sync
+job_chain[:jobs] << { job: CanvasSync::Jobs::ForkGather, options: {} }
+
+# Add a custom job to the end of the chain. Custom jobs must accept 2 arguments (job_chain, options) and call CanvasSync.invoke_next(job_chain) when complete
+job_chain[:jobs] << { job: CanvasSyncCompleteWorker, options: { job_id: job.id } }
+
+CanvasSync.invoke_next(job_chain)
+```
+
 ### Processor
 
 Your processor class must implement a `process` class method that receives a `report_file_path` and a hash of `options`. (See the `CanvasSync::Processors::ProvisioningReportProcessor` for an example.) The gem handles the work of enqueueing and downloading the report and then passes the file path to your class to process as needed. A simple example might be:
@@ -141,7 +164,7 @@ The `CanvasSync.process_jobs` method allows you to pass in a chain of jobs to ru
 
 Here is an example that runs our new report job first followed by the builtin provisioning job:
 
-```
+```ruby
 job_chain = {
   jobs: [
     { job: MyReallyCoolReportJob, options: {} },
@@ -156,7 +179,7 @@ CanvasSync.process_jobs(job_chain)
 What if you've got some other job that you want run that doesn't deal with a report? No problem! Just make sure you call `CanvasSync.invoke_next` at the end of your job. Example:
 
 
-```
+```ruby
 class SomeRandomJob < CanvasSync::Job
   def perform(job_chain, options)
     i_dunno_do_something!

data/db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb

@@ -0,0 +1,13 @@
+if Rails.version.to_f >= 5.0
+  class AddForkCountToCanvasSyncJobLogs < ActiveRecord::Migration[Rails.version.to_f]
+    def change
+      add_column :canvas_sync_job_logs, :fork_count, :integer
+    end
+  end
+else
+  class AddForkCountToCanvasSyncJobLogs < ActiveRecord::Migration
+    def change
+      add_column :canvas_sync_job_logs, :fork_count, :integer
+    end
+  end
+end

data/lib/canvas_sync.rb

@@ -6,6 +6,7 @@ require "canvas_sync/sidekiq_job"
 require "canvas_sync/api_syncable"
 require "canvas_sync/jobs/report_starter"
 require "canvas_sync/jobs/report_checker"
+require "canvas_sync/jobs/fork_gather"
 require "canvas_sync/jobs/report_processor_job"
 require "canvas_sync/jobs/sync_provisioning_report_job"
 require "canvas_sync/jobs/sync_simple_table_job.rb"
@@ -118,6 +119,18 @@ module CanvasSync
       next_job_class.perform_later(duped_job_chain, next_job[:options])
     end
 
+    def fork(job_log, job_chain, keys: [])
+      duped_job_chain = Marshal.load(Marshal.dump(job_chain))
+      duped_job_chain[:global_options][:fork_path] ||= []
+      duped_job_chain[:global_options][:fork_keys] ||= []
+      duped_job_chain[:global_options][:fork_path] << job_log.job_id
+      duped_job_chain[:global_options][:fork_keys] << ['canvas_term_id']
+      sub_items = yield duped_job_chain
+      sub_count =  sub_items.respond_to?(:count) ? sub_items.count : sub_items
+      job_log.update!(fork_count: sub_count)
+      sub_items
+    end
+
     # Syn any report to an specific set of models
     #
     # @param reports_mapping [Array<Hash>] List of reports with their specific model and params

data/lib/canvas_sync/jobs/fork_gather.rb

@@ -0,0 +1,26 @@
+module CanvasSync
+  module Jobs
+    class ForkGather < CanvasSync::Job
+      def perform(job_chain, options)
+        fork_item = (job_chain[:global_options][:fork_path] || []).pop
+
+        if fork_item.present?
+          forked_job = CanvasSync::JobLog.find_by(job_id: fork_item)
+          forked_job.with_lock do
+            forked_job.fork_count -= 1
+            forked_job.save!
+          end
+
+          if forked_job.fork_count <= 0
+            (job_chain[:global_options][:fork_keys] || []).pop&.each do |k|
+              job_chain[:global_options].delete(k.to_sym)
+            end
+            CanvasSync.invoke_next(job_chain)
+          end
+        else
+          CanvasSync.invoke_next(job_chain)
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -8,13 +8,23 @@ module CanvasSync
       #   models to sync.
       def perform(job_chain, options)
         if options[:term_scope]
-          Term.send(options[:term_scope]).find_each do |term|
-            # Deep copy the job_chain so each report gets the correct term id passed into
-            # its options with no side effects
-            term_id = get_term_id(term)
-            duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-            duped_job_chain[:global_options][:canvas_term_id] = term_id
-            start_report(report_params(options, term_id), duped_job_chain, options)
+          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |job_chain|
+            Term.send(options[:term_scope]).find_each.map do |term|
+              # Deep copy the job_chain so each report gets the correct term id passed into
+              # its options with no side effects
+              term_id = get_term_id(term)
+              duped_job_chain = Marshal.load(Marshal.dump(job_chain))
+              duped_job_chain[:global_options][:canvas_term_id] = term_id
+              {
+                job_chain: duped_job_chain,
+                params: report_params(options, term_id),
+                options: options,
+              }
+            end
+          end
+
+          sub_reports.each do |r|
+            start_report(r[:params], r[:job_chain], r[:options])
           end
         else
           start_report(report_params(options), job_chain, options)

data/lib/canvas_sync/jobs/sync_simple_table_job.rb

@@ -9,13 +9,23 @@ module CanvasSync
       # @param options [Hash]
       def perform(job_chain, options)
         if options[:term_scope]
-          Term.send(options[:term_scope]).find_each do |term|
-            # Deep copy the job_chain so each report gets the correct term id passed into
-            # its options with no side effects
-            term_id = get_term_id(term)
-            duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-            duped_job_chain[:global_options][:canvas_term_id] = term_id
-            start_report(report_params(options, term_id), duped_job_chain, options)
+          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |job_chain|
+            Term.send(options[:term_scope]).find_each.map do |term|
+              # Deep copy the job_chain so each report gets the correct term id passed into
+              # its options with no side effects
+              term_id = get_term_id(term)
+              duped_job_chain = Marshal.load(Marshal.dump(job_chain))
+              duped_job_chain[:global_options][:canvas_term_id] = term_id
+              {
+                job_chain: duped_job_chain,
+                params: report_params(options, term_id),
+                options: options,
+              }
+            end
+          end
+
+          sub_reports.each do |r|
+            start_report(r[:params], r[:job_chain], r[:options])
           end
         else
           start_report(report_params(options), job_chain, options)

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.10.4".freeze
+  VERSION = "0.10.5".freeze
 end

data/spec/canvas_sync/jobs/fork_gather_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::Jobs::ForkGather do
+  describe '#perform' do
+    let!(:job_log) { CanvasSync::JobLog.create!(job_id: 'BLAH', fork_count: 3) }
+    let(:job_chain) { {jobs: [], global_options: { fork_path: ['BLAH'] }} }
+
+    it 'decrements fork_count' do
+      CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
+      expect(job_log.reload.fork_count).to eq 2
+    end
+
+    it 'does not continue chain if fork_count > 0' do
+      expect(CanvasSync).not_to receive(:invoke_next)
+      CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
+    end
+
+    it 'continues the chain if fork_count <= 0' do
+      job_log.update!(fork_count: 1)
+      expect(CanvasSync).to receive(:invoke_next)
+      CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
+    end
+
+    it 'continues if no fork_path is specified' do
+      job_chain[:global_options].delete(:fork_path)
+      expect(CanvasSync).to receive(:invoke_next)
+      CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
+    end
+  end
+end

data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb

@@ -12,6 +12,7 @@ RSpec.describe CanvasSync::Jobs::SyncProvisioningReportJob do
       it 'enqueues a ReportStarter for a provisioning report for the specified models for each term' do
         expected_job_chain = Marshal.load(Marshal.dump(job_chain))
         expected_job_chain[:global_options][:canvas_term_id] = active_term_1.canvas_id
+        expected_job_chain[:global_options] = hash_including(expected_job_chain[:global_options])
 
         expect(CanvasSync::Jobs::ReportStarter).to receive(:perform_later)
           .with(
@@ -31,6 +32,7 @@ RSpec.describe CanvasSync::Jobs::SyncProvisioningReportJob do
 
         expected_job_chain_2 = Marshal.load(Marshal.dump(job_chain))
         expected_job_chain_2[:global_options][:canvas_term_id] = active_term_2.canvas_id
+        expected_job_chain_2[:global_options] = hash_including(expected_job_chain_2[:global_options])
 
         expect(CanvasSync::Jobs::ReportStarter).to receive(:perform_later)
           .with(
@@ -52,6 +54,7 @@ RSpec.describe CanvasSync::Jobs::SyncProvisioningReportJob do
           job_chain,
           { models: ['users', 'courses'], term_scope: 'active' }
         )
+        expect(CanvasSync::JobLog.last.fork_count).to eq 2
       end
     end
 

data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb

@@ -11,6 +11,7 @@ RSpec.describe CanvasSync::Jobs::SyncSimpleTableJob do
       it 'enqueues a ReportStarter for a provisioning report for the specified model for a term' do
         expected_job_chain = Marshal.load(Marshal.dump(job_chain))
         expected_job_chain[:global_options][:canvas_term_id] = active_term_1.canvas_id
+        expected_job_chain[:global_options] = hash_including(expected_job_chain[:global_options])
 
         expect(CanvasSync::Jobs::ReportStarter).to receive(:perform_later)
           .with(

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20190702203632) do
+ActiveRecord::Schema.define(version: 20190916154829) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -92,6 +92,7 @@ ActiveRecord::Schema.define(version: 20190702203632) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.string "job_id"
+    t.integer "fork_count"
     t.index ["job_id"], name: "index_canvas_sync_job_logs_on_job_id"
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.10.4
+  version: 0.10.5
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-11 00:00:00.000000000 Z
+date: 2019-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -319,6 +319,7 @@ files:
 - config/initializers/apartment.rb
 - db/migrate/20170915210836_create_canvas_sync_job_log.rb
 - db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb
+- db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb
 - lib/canvas_sync.rb
 - lib/canvas_sync/api_syncable.rb
 - lib/canvas_sync/config.rb
@@ -366,6 +367,7 @@ files:
 - lib/canvas_sync/importers/bulk_importer.rb
 - lib/canvas_sync/importers/legacy_importer.rb
 - lib/canvas_sync/job.rb
+- lib/canvas_sync/jobs/fork_gather.rb
 - lib/canvas_sync/jobs/report_checker.rb
 - lib/canvas_sync/jobs/report_processor_job.rb
 - lib/canvas_sync/jobs/report_starter.rb
@@ -392,6 +394,7 @@ files:
 - lib/canvas_sync/sidekiq_job.rb
 - lib/canvas_sync/version.rb
 - spec/canvas_sync/canvas_sync_spec.rb
+- spec/canvas_sync/jobs/fork_gather_spec.rb
 - spec/canvas_sync/jobs/job_spec.rb
 - spec/canvas_sync/jobs/report_checker_spec.rb
 - spec/canvas_sync/jobs/report_processor_job_spec.rb
@@ -562,6 +565,7 @@ specification_version: 4
 summary: Gem for generating Canvas models and migrations and syncing data from Canvas
 test_files:
 - spec/canvas_sync/canvas_sync_spec.rb
+- spec/canvas_sync/jobs/fork_gather_spec.rb
 - spec/canvas_sync/jobs/job_spec.rb
 - spec/canvas_sync/jobs/report_checker_spec.rb
 - spec/canvas_sync/jobs/report_processor_job_spec.rb