canvas_sync 0.16.5 → 0.17.0.beta5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9be3b81c22d5b45a02d88fa29b5d57afa152ee7dec6ce0ecf3d81c041507c619
-  data.tar.gz: 0c42577064cbf018f8fd8d5f665e1adfbc542e38e704a675e735491543b3ab40
+  metadata.gz: eba7624a65721ad2183b188372b167c47bde879c37c595ef13398e4ec4b835b5
+  data.tar.gz: a69c72c000c782bff566016ca1cc7813c0c1d6829f2516291721672e3ee7d5ec
 SHA512:
-  metadata.gz: 9ea295b8cc43aa6b27bf6067cf684d024430a9fd85608bdc0a243113920b1b65b72c257e6a99644afb5d20455ecf2265949ce941a9ae1d9bea160ab1bee803ba
-  data.tar.gz: e63b7f557e792cb850a5799753f9effe436ad391bc6e4442768683b8c4d7c8eea1d1e9bbd6419127f2f4401f6cce6129019fb73192496f7efc1d731d7ef48847
+  metadata.gz: 00e85f161794e1526cf734aabaaf364b55b92b93a832248bd8f17b4eeb97081e20fc8ef7e84ae7029dbd654ad46410939d10ec4db5ba79394b9326de49948342
+  data.tar.gz: d30bb87abb822cd0f5ad10b60b6941326ac8a2e964241fa6e8403c5a5bf303e3d6945bd79ff500a181bd21a6805161281f4fe05fdd091f42b295ebf820772362

data/README.md

@@ -91,27 +91,45 @@ This gem also helps with syncing and processing other reports if needed. In orde
 - Integrate your reports with the `ReportStarter`
 - Tell the gem what jobs to run
 
+### `updated_after`
+An `updated_after` param may be passed when triggering a provision or making a chain:
+```ruby
+CanvasSync.default_provisioning_report_chain(
+  %i[list of models to sync], updated_after: false
+)
+```
+It may be one of the following values:
+* `false` - Will not apply any `updated_after` filtering to the requested reports
+* An ISO-8601 Date - Will pass the supplied date ad the `updated_after` param for the requested reports
+* `true` (Default) - Will use the start date of the last successful sync
+
 ### Extensible chain
 It is sometimes desired to extend or customize the chain of jobs that are run with CanvasSync.
 This can be achieved with the following pattern:
 
 ```ruby
-job_chain = CanvasSync.default_provisioning_report_chain(
+chain = CanvasSync.default_provisioning_report_chain(
   %i[list of models to sync]
 )
 
-# CanvasSync forks the chain for each term within a term scope. The ForkGather Job can be used to unfork the chain.
-# For example multiple Terms are in the term_scope. CanvasSync syncs Accounts, Terms, and Users (if enabled) and then
-# forks the chain to sync other models (eg Course, Sections, etc.) per-Term.
-# ForkGather will wait until all the forked chains are complete before continuing.
-# TL;DR: Jobs placed after SyncProvisioningReportJob and before ForkGather will run once per Term per Sync;
-# Jobs placed before SyncProvisioningReportJob or after ForkGather will run once per Sync
-job_chain[:jobs] << { job: CanvasSync::Jobs::ForkGather, options: {} }
+# Add a custom job to the end of the chain.
+chain << { job: CanvasSyncCompleteWorker, parameters: [{ job_id: job.id }] }
+chain << { job: CanvasSyncCompleteWorker, options: { job_id: job.id } } # If an options key is provided, it will be automatically appended to the end of the :parameters array
+
+chain.process!
 
-# Add a custom job to the end of the chain. Custom jobs must accept 2 arguments (job_chain, options) and call CanvasSync.invoke_next(job_chain) when complete
-job_chain[:jobs] << { job: CanvasSyncCompleteWorker, options: { job_id: job.id } }
+# The chain object provides a fairly extensive API:
+chain.insert({ job: SomeOtherJob }) # Adds the job to the end of the chain
+chain.insert_at(0, { job: SomeOtherJob }) # Adds the job to the beginning of the chain
+chain.insert({ job: SomeOtherJob }, after: 'CanvasSync::Jobs::SyncTermsJob') # Adds the job right after the SyncTermsJob
+chain.insert({ job: SomeOtherJob }, before: 'CanvasSync::Jobs::SyncTermsJob') # Adds the job right before the SyncTermsJob
+chain.insert({ job: SomeOtherJob }, with: 'CanvasSync::Jobs::SyncTermsJob') # Adds the job to be performed concurrently with the SyncTermsJob
 
-CanvasSync.invoke_next(job_chain)
+# Some Jobs (such as the SyncTermsJob) have a sub-chain for, eg, Courses.
+# chain.insert is aware of these sub-chains and will recurse into them when looking for a before:/after:/with: reference
+chain.insert({ job: SomeOtherJob }, after: 'CanvasSync::Jobs::SyncCoursesJob') # Adds the job to be performed after SyncCoursesJob (which is a sub-job of the terms job and is duplicated for each term in the term_scope:)
+# You can also retrieve the sub-chain like so:
+chain.get_sub_chain('CanvasSync::Jobs::SyncTermsJob')
 ```
 
 ### Processor
@@ -134,9 +152,8 @@ Let's say we have a custom Canvas report called "my_really_cool_report_csv". Fir
 
 ```ruby
 class MyReallyCoolReportJob < CanvasSync::Jobs::ReportStarter
-  def perform(job_chain, options)
+  def perform(options)
     super(
-      job_chain,
       'my_really_cool_report_csv', # Report name
       { "parameters[param1]" => true }, # Report parameters
       MyCoolProcessor.to_s, # Your processor class as a string
@@ -148,57 +165,6 @@ end
 
 You can also see examples in `lib/canvas_sync/jobs/sync_users_job.rb` and `lib/canvas_sync/jobs/sync_provisioning_report.rb`.
 
-### Start the jobs
-
-The `CanvasSync.process_jobs` method allows you to pass in a chain of jobs to run. The job chain must be formatted like:
-
-```ruby
-{
-  jobs: [
-    { job: JobClass, options: {} },
-    { job: JobClass2, options: {} }
-  ],
-  global_options: {}
-}
-```
-
-Here is an example that runs our new report job first followed by the builtin provisioning job:
-
-```ruby
-job_chain = {
-  jobs: [
-    { job: MyReallyCoolReportJob, options: {} },
-    { job: CanvasSync::Jobs::SyncProvisioningReportJob, options: { models: ['users', 'courses'] } }
-  ],
-  global_options: {}
-}
-
-CanvasSync.process_jobs(job_chain)
-```
-
-What if you've got some other job that you want run that doesn't deal with a report? No problem! Just make sure you call `CanvasSync.invoke_next` at the end of your job. Example:
-
-
-```ruby
-class SomeRandomJob < CanvasSync::Job
-  def perform(job_chain, options)
-    i_dunno_do_something!
-
-    CanvasSync.invoke_next(job_chain)
-  end
-end
-
-job_chain = {
-  jobs: [
-    { job: SomeRandomJob, options: {} },
-    { job: CanvasSync::Jobs::SyncProvisioningReportJob, options: { models: ['users', 'courses'] } }
-  ],
-  global_options: {}
-}
-
-CanvasSync.process_jobs(job_chain)
-```
-
 ### Batching
 
 The provisioning report uses the `CanvasSync::Importers::BulkImporter` class to bulk import rows with the activerecord-import gem. It inserts rows in batches of 10,000 by default. This can be customized by setting the `BULK_IMPORTER_BATCH_SIZE` environment variable.
@@ -257,6 +223,14 @@ class CanvasSyncModel < ApplicationRecord
 end
 ```
 
+### Job Batching
+CanvasSync adds a `CanvasSync::JobBatches` module. It adds Sidekiq/sidekiq-batch like support for Job Batches.
+It integrates automatically with both Sidekiq and ActiveJob. The API is highly similar to the Sidekiq-batch implementation,
+documentation for which can be found at https://github.com/mperham/sidekiq/wiki/Batches
+
+A batch can be created using `Sidekiq::Batch` or `CanvasSync::JobBatching::Batch`.
+
+Also see `canvas_sync/jobs/begin_sync_chain_job`, `canvas_sync/Job_batches/jobs/serial_batch_job`, or `canvas_sync/Job_batches/jobs/concurrent_batch_job` for example usage.
 
 ## Legacy Support
 
@@ -308,80 +282,19 @@ end
 ## Syncronize different reports
 CanvasSync provides the functionality to import data from other reports into an specific table.
 
-This can be achived by using the followin method
-
-```ruby
-CanvasSync.provisioning_sync(<array of models to sync>, term_scope: <optional term scope>)
-CanvasSync
-  .simple_report_sync(
-    [
-      {
-        report_name: <report name>,
-        model: <model to sync>,
-        params: <hash with the require parameters the report needs to sync>
-      },
-      {
-        report_name: <report name>,
-        model: <model to sync>,
-        params: <hash with the require parameters the report needs to sync>
-      },
-      ...
-    ],
-    term_scope: <optional term scope>
-  )
-```
-
-Example:
+This can be achieved by using the following method
 
 ```ruby
-CanvasSync
-  .simple_report_sync(
-    [
-      {
-        report_name: 'proservices_provisioning_csv',
-        model: 'users',
-        params: {
-          "parameters[include_deleted]" => true,
-          "parameters[users]" => true
-        }
-      },
-      {
-        report_name: 'proservices_provisioning_csv',
-        model: 'accounts',
-        params: {
-          "parameters[include_deleted]" => true,
-          "parameters[accounts]" => true
-        }
-      }
-    ]
-  )
-```
-
-Example with the term_scope active:
-
-```ruby
-CanvasSync
-  .simple_report_sync(
-    [
-      {
-        report_name: 'proservices_provisioning_csv',
-        model: 'sections',
-        params: {
-          "parameters[include_deleted]" => true,
-          "parameters[sections]" => true
-        }
-      },
-      {
-        report_name: 'proservices_provisioning_csv',
-        model: 'courses',
-        params: {
-          "parameters[include_deleted]" => true,
-          "parameters[courses]" => true
-        }
-      }
-    ],
-    term_scope: 'active'
-  )
+chain = CanvasSync.default_provisioning_report_chain
+chain << {
+  job: CanvasSync::Jobs::SyncSimpleTableJob,
+  options: {
+    report_name: <report name>,
+    model: <model to sync>,
+    params: <hash with the require parameters the report needs to sync>
+  },
+}
+chain.process!
 ```
 
 ## Configuration
@@ -421,7 +334,6 @@ class CanvasSyncStarterWorker
         }
       }
     )
-    CanvasSync.invoke_next(job_chain)
   end
 
   def self.handle_canvas_sync_error(error, **options)

data/app/models/canvas_sync/sync_batch.rb

@@ -0,0 +1,5 @@
+module CanvasSync
+  class SyncBatch < ApplicationRecord
+    serialize :job_arguments, Array
+  end
+end

data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb

@@ -0,0 +1,11 @@
+class CreateCanvasSyncSyncBatches < CanvasSync::MiscHelper::MigrationClass
+  def change
+    create_table :canvas_sync_sync_batches do |t|
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.string :status
+
+      t.timestamps
+    end
+  end
+end

data/lib/canvas_sync.rb

@@ -5,7 +5,6 @@ require "canvas_sync/engine"
 require "canvas_sync/misc_helper"
 require "canvas_sync/class_callback_executor"
 require "canvas_sync/job"
-require "canvas_sync/job_chain"
 require "canvas_sync/sidekiq_job"
 require "canvas_sync/api_syncable"
 require "canvas_sync/record"
@@ -14,6 +13,8 @@ require "canvas_sync/jobs/report_checker"
 require "canvas_sync/jobs/report_processor_job"
 require "canvas_sync/config"
 
+require "canvas_sync/job_batches/batch"
+
 Dir[File.dirname(__FILE__) +  "/canvas_sync/jobs/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/processors/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/importers/*.rb"].each { |file| require file }
@@ -58,6 +59,9 @@ module CanvasSync
     graded_submissions
   ].freeze
 
+  JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::SyncTermsJob, :sub_jobs)
+  JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::BeginSyncChainJob, 0)
+
   class << self
     # Runs a standard provisioning sync job with no extra report types.
     # Terms will be synced first using the API. If you are syncing users/roles/admins
@@ -73,51 +77,9 @@ module CanvasSync
     #   and inserts it into the database. If an array of model names is provided then only those models will use legacy support.
     # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
     #   canvas_sync_client methods require an account ID.
-    def provisioning_sync(models, term_scope: nil, legacy_support: false, account_id: nil)
+    def provisioning_sync(models, **kwargs)
       validate_models!(models)
-      invoke_next(default_provisioning_report_chain(models, term_scope, legacy_support, account_id))
-    end
-
-    # Runs a report different from provisioning sync job with no extra report types.
-    #
-    # @param reports_mapping [Array<Hash>] An Array of hash that list the model and params with their report you
-    #   want to import:
-    #   [{model: 'submissions', report_name: 'my_report_name_csv', params: { "parameters[include_deleted]" => true } }, ...]
-    # @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model.
-    #   The provisioning report will be run for each of the terms contained in that scope.
-    # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
-    #   canvas_sync_client methods require an account ID.
-    def simple_report_sync(reports_mapping, term_scope: nil, account_id: nil)
-      invoke_next(simple_report_chain(reports_mapping, term_scope, account_id))
-    end
-
-    # Runs a chain of ordered jobs
-    #
-    # See the README for usage and examples
-    #
-    # @param job_chain [Hash]
-    def process_jobs(job_chain)
-      invoke_next(job_chain)
-    end
-
-    # @deprecated
-    def duplicate_chain(job_chain)
-      Marshal.load(Marshal.dump(job_chain))
-    end
-
-    # Invokes the next job in a chain of jobs.
-    #
-    # This should typically be called automatically by the gem where necessary.
-    #
-    # @param job_chain [Hash] A chain of jobs to execute
-    def invoke_next(job_chain, extra_options: {})
-      job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
-      job_chain.perform_next(extra_options)
-    end
-
-    def fork(job_log, job_chain, keys: [], &blk)
-      job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
-      job_chain.fork(job_log, keys: keys, &blk)
+      default_provisioning_report_chain(models, **kwargs).process!
     end
 
     # Given a Model or Relation, scope it down to items that should be synced
@@ -134,33 +96,6 @@ module CanvasSync
       scope
     end
 
-    # Syn any report to an specific set of models
-    #
-    # @param reports_mapping [Array<Hash>] List of reports with their specific model and params
-    # @param term_scope [String]
-    # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
-    #   canvas_sync_client methods require an account ID.
-    def simple_report_chain(reports_mapping, term_scope=nil, account_id=nil)
-      jobs = reports_mapping.map do |report|
-        {
-          job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
-          options: {
-            report_name: report[:report_name],
-            model: report[:model],
-            mapping: report[:model],
-            klass: report[:model].singularize.capitalize.to_s,
-            term_scope: term_scope,
-            params: report[:params]
-          }
-        }
-      end
-
-      global_options = {}
-      global_options[:account_id] = account_id if account_id.present?
-
-      JobChain.new(jobs: jobs, global_options: global_options)
-    end
-
     # Syncs terms, users/roles/admins if necessary, then the rest of the specified models.
     #
     # @param models [Array<String>]
@@ -171,7 +106,7 @@ module CanvasSync
     # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
     #   canvas_sync_client methods require an account ID.
     # @return [Hash]
-    def default_provisioning_report_chain(models, term_scope=nil, legacy_support=false, account_id=nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
+    def default_provisioning_report_chain(models, term_scope: nil, legacy_support: false, account_id: nil, updated_after: nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
       return unless models.present?
       models.map! &:to_s
       term_scope = term_scope.to_s if term_scope
@@ -190,64 +125,67 @@ module CanvasSync
         context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
       }.with_indifferent_access
 
-      jobs = []
+      root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
+      concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
+      root_chain << concurrent_root_chain
+      current_chain = concurrent_root_chain
+
       try_add_model_job = ->(model) {
         return unless models.include?(model)
-        jobs.push(job: model_job_map[model].to_s, options: options[model.to_sym] || {})
+        current_chain << { job: model_job_map[model].to_s, options: options[model.to_sym] || {} }
         models -= [model]
       }
 
       ##############################
-      # Pre provisioning report jobs
+      # General provisioning jobs (not term-scoped)
       ##############################
 
-      # Always sync Terms first
-      models.unshift('terms') unless models.include?('terms')
-      try_add_model_job.call('terms')
+      # Terms are always synced regardless of the models option
+      models -= ['terms']
 
-      # Accounts, users, roles, and admins are synced before provisioning because they cannot be scoped to term
+      # Accounts, users, roles, and admins cannot be scoped to term
       try_add_model_job.call('accounts')
 
       # These Models use the provisioning report, but are not term-scoped,
-      # so we sync them before to ensure work is not duplicated
+      # so we sync them outside of the term scoping to ensure work is not duplicated
       if term_scope.present?
         models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
-        jobs.concat(
-          generate_provisioning_jobs(first_provisioning_models, options)
-        )
+        current_chain.insert(generate_provisioning_jobs(first_provisioning_models, options))
       end
 
       try_add_model_job.call('roles')
       try_add_model_job.call('admins')
-      pre_provisioning_jobs = jobs
 
       ###############################
-      # Post provisioning report jobs
+      # Per-term provisioning jobs
       ###############################
 
-      jobs = []
+      per_term_chain = JobBatches::ChainBuilder.new(model_job_map[:terms])
+      per_term_chain.params[:term_scope] = term_scope
+      current_chain << per_term_chain
+      current_chain = per_term_chain
+
       try_add_model_job.call('assignments')
       try_add_model_job.call('submissions')
       try_add_model_job.call('assignment_groups')
       try_add_model_job.call('context_modules')
       try_add_model_job.call('context_module_items')
-      post_provisioning_jobs = jobs
+
+      current_chain.insert(
+        generate_provisioning_jobs(models, options, only_split: ['users'])
+      )
 
       ###############################
-      # Main provisioning job and queueing
+      # Wrap it all up
       ###############################
 
-      jobs = [
-        *pre_provisioning_jobs,
-        *generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']),
-        *post_provisioning_jobs,
-      ]
-
-      global_options = { legacy_support: legacy_support }
+      global_options = { legacy_support: legacy_support, updated_after: updated_after }
       global_options[:account_id] = account_id if account_id.present?
       global_options.merge!(options[:global]) if options[:global].present?
 
-      JobChain.new(jobs: jobs, global_options: global_options)
+      root_chain.params[1] = global_options
+
+      root_chain
     end
 
     def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)