canvas_sync 0.16.5 → 0.17.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 9be3b81c22d5b45a02d88fa29b5d57afa152ee7dec6ce0ecf3d81c041507c619
-  data.tar.gz: 0c42577064cbf018f8fd8d5f665e1adfbc542e38e704a675e735491543b3ab40
+SHA1:
+  metadata.gz: b4ffe8d3a42b647f8c30f1d831c4a86998bc7c1b
+  data.tar.gz: 2ba762a479d2df35e22a69b7930d051a2e43bd30
 SHA512:
-  metadata.gz: 9ea295b8cc43aa6b27bf6067cf684d024430a9fd85608bdc0a243113920b1b65b72c257e6a99644afb5d20455ecf2265949ce941a9ae1d9bea160ab1bee803ba
-  data.tar.gz: e63b7f557e792cb850a5799753f9effe436ad391bc6e4442768683b8c4d7c8eea1d1e9bbd6419127f2f4401f6cce6129019fb73192496f7efc1d731d7ef48847
+  metadata.gz: 6702f6754b217edfe2c7e4ff51c586940e7073993b5e3a79464e49308edd274af9bc8165f38aa919ef3c288ec86f1623065a91f42121f68c4da1389f27a57cb2
+  data.tar.gz: 71365002bcf14b762b00b73ae805862d8691a2b78ecd0254c0ef7fcf56eea6caeae838652e7ca4a685834eb380a08e782cf78e591b5af2c8950b4d4fd763e704

data/README.md

@@ -91,27 +91,45 @@ This gem also helps with syncing and processing other reports if needed. In orde
 - Integrate your reports with the `ReportStarter`
 - Tell the gem what jobs to run
 
+### `updated_after`
+An `updated_after` param may be passed when triggering a provision or making a chain:
+```ruby
+CanvasSync.default_provisioning_report_chain(
+  %i[list of models to sync], updated_after: false
+)
+```
+It may be one of the following values:
+* `false` - Will not apply any `updated_after` filtering to the requested reports
+* An ISO-8601 Date - Will pass the supplied date ad the `updated_after` param for the requested reports
+* `true` (Default) - Will use the start date of the last successful sync
+
 ### Extensible chain
 It is sometimes desired to extend or customize the chain of jobs that are run with CanvasSync.
 This can be achieved with the following pattern:
 
 ```ruby
-job_chain = CanvasSync.default_provisioning_report_chain(
+chain = CanvasSync.default_provisioning_report_chain(
   %i[list of models to sync]
 )
 
-# CanvasSync forks the chain for each term within a term scope. The ForkGather Job can be used to unfork the chain.
-# For example multiple Terms are in the term_scope. CanvasSync syncs Accounts, Terms, and Users (if enabled) and then
-# forks the chain to sync other models (eg Course, Sections, etc.) per-Term.
-# ForkGather will wait until all the forked chains are complete before continuing.
-# TL;DR: Jobs placed after SyncProvisioningReportJob and before ForkGather will run once per Term per Sync;
-# Jobs placed before SyncProvisioningReportJob or after ForkGather will run once per Sync
-job_chain[:jobs] << { job: CanvasSync::Jobs::ForkGather, options: {} }
+# Add a custom job to the end of the chain.
+chain << { job: CanvasSyncCompleteWorker, parameters: [{ job_id: job.id }] }
+chain << { job: CanvasSyncCompleteWorker, options: { job_id: job.id } } # If an options key is provided, it will be automatically appended to the end of the :parameters array
+
+chain.process!
 
-# Add a custom job to the end of the chain. Custom jobs must accept 2 arguments (job_chain, options) and call CanvasSync.invoke_next(job_chain) when complete
-job_chain[:jobs] << { job: CanvasSyncCompleteWorker, options: { job_id: job.id } }
+# The chain object provides a fairly extensive API:
+chain.insert({ job: SomeOtherJob }) # Adds the job to the end of the chain
+chain.insert_at(0, { job: SomeOtherJob }) # Adds the job to the beginning of the chain
+chain.insert({ job: SomeOtherJob }, after: 'CanvasSync::Jobs::SyncTermsJob') # Adds the job right after the SyncTermsJob
+chain.insert({ job: SomeOtherJob }, before: 'CanvasSync::Jobs::SyncTermsJob') # Adds the job right before the SyncTermsJob
+chain.insert({ job: SomeOtherJob }, with: 'CanvasSync::Jobs::SyncTermsJob') # Adds the job to be performed concurrently with the SyncTermsJob
 
-CanvasSync.invoke_next(job_chain)
+# Some Jobs (such as the SyncTermsJob) have a sub-chain for, eg, Courses.
+# chain.insert is aware of these sub-chains and will recurse into them when looking for a before:/after:/with: reference
+chain.insert({ job: SomeOtherJob }, after: 'CanvasSync::Jobs::SyncCoursesJob') # Adds the job to be performed after SyncCoursesJob (which is a sub-job of the terms job and is duplicated for each term in the term_scope:)
+# You can also retrieve the sub-chain like so:
+chain.get_sub_chain('CanvasSync::Jobs::SyncTermsJob')
 ```
 
 ### Processor
@@ -134,9 +152,8 @@ Let's say we have a custom Canvas report called "my_really_cool_report_csv". Fir
 
 ```ruby
 class MyReallyCoolReportJob < CanvasSync::Jobs::ReportStarter
-  def perform(job_chain, options)
+  def perform(options)
     super(
-      job_chain,
       'my_really_cool_report_csv', # Report name
       { "parameters[param1]" => true }, # Report parameters
       MyCoolProcessor.to_s, # Your processor class as a string
@@ -148,57 +165,6 @@ end
 
 You can also see examples in `lib/canvas_sync/jobs/sync_users_job.rb` and `lib/canvas_sync/jobs/sync_provisioning_report.rb`.
 
-### Start the jobs
-
-The `CanvasSync.process_jobs` method allows you to pass in a chain of jobs to run. The job chain must be formatted like:
-
-```ruby
-{
-  jobs: [
-    { job: JobClass, options: {} },
-    { job: JobClass2, options: {} }
-  ],
-  global_options: {}
-}
-```
-
-Here is an example that runs our new report job first followed by the builtin provisioning job:
-
-```ruby
-job_chain = {
-  jobs: [
-    { job: MyReallyCoolReportJob, options: {} },
-    { job: CanvasSync::Jobs::SyncProvisioningReportJob, options: { models: ['users', 'courses'] } }
-  ],
-  global_options: {}
-}
-
-CanvasSync.process_jobs(job_chain)
-```
-
-What if you've got some other job that you want run that doesn't deal with a report? No problem! Just make sure you call `CanvasSync.invoke_next` at the end of your job. Example:
-
-
-```ruby
-class SomeRandomJob < CanvasSync::Job
-  def perform(job_chain, options)
-    i_dunno_do_something!
-
-    CanvasSync.invoke_next(job_chain)
-  end
-end
-
-job_chain = {
-  jobs: [
-    { job: SomeRandomJob, options: {} },
-    { job: CanvasSync::Jobs::SyncProvisioningReportJob, options: { models: ['users', 'courses'] } }
-  ],
-  global_options: {}
-}
-
-CanvasSync.process_jobs(job_chain)
-```
-
 ### Batching
 
 The provisioning report uses the `CanvasSync::Importers::BulkImporter` class to bulk import rows with the activerecord-import gem. It inserts rows in batches of 10,000 by default. This can be customized by setting the `BULK_IMPORTER_BATCH_SIZE` environment variable.
@@ -257,6 +223,14 @@ class CanvasSyncModel < ApplicationRecord
 end
 ```
 
+### Job Batching
+CanvasSync adds a `CanvasSync::JobBatches` module. It adds Sidekiq/sidekiq-batch like support for Job Batches.
+It integrates automatically with both Sidekiq and ActiveJob. The API is highly similar to the Sidekiq-batch implementation,
+documentation for which can be found at https://github.com/mperham/sidekiq/wiki/Batches
+
+A batch can be created using `Sidekiq::Batch` or `CanvasSync::JobBatching::Batch`.
+
+Also see `canvas_sync/jobs/begin_sync_chain_job`, `canvas_sync/Job_batches/jobs/serial_batch_job`, or `canvas_sync/Job_batches/jobs/concurrent_batch_job` for example usage.
 
 ## Legacy Support
 
@@ -308,80 +282,19 @@ end
 ## Syncronize different reports
 CanvasSync provides the functionality to import data from other reports into an specific table.
 
-This can be achived by using the followin method
-
-```ruby
-CanvasSync.provisioning_sync(<array of models to sync>, term_scope: <optional term scope>)
-CanvasSync
-  .simple_report_sync(
-    [
-      {
-        report_name: <report name>,
-        model: <model to sync>,
-        params: <hash with the require parameters the report needs to sync>
-      },
-      {
-        report_name: <report name>,
-        model: <model to sync>,
-        params: <hash with the require parameters the report needs to sync>
-      },
-      ...
-    ],
-    term_scope: <optional term scope>
-  )
-```
-
-Example:
+This can be achieved by using the following method
 
 ```ruby
-CanvasSync
-  .simple_report_sync(
-    [
-      {
-        report_name: 'proservices_provisioning_csv',
-        model: 'users',
-        params: {
-          "parameters[include_deleted]" => true,
-          "parameters[users]" => true
-        }
-      },
-      {
-        report_name: 'proservices_provisioning_csv',
-        model: 'accounts',
-        params: {
-          "parameters[include_deleted]" => true,
-          "parameters[accounts]" => true
-        }
-      }
-    ]
-  )
-```
-
-Example with the term_scope active:
-
-```ruby
-CanvasSync
-  .simple_report_sync(
-    [
-      {
-        report_name: 'proservices_provisioning_csv',
-        model: 'sections',
-        params: {
-          "parameters[include_deleted]" => true,
-          "parameters[sections]" => true
-        }
-      },
-      {
-        report_name: 'proservices_provisioning_csv',
-        model: 'courses',
-        params: {
-          "parameters[include_deleted]" => true,
-          "parameters[courses]" => true
-        }
-      }
-    ],
-    term_scope: 'active'
-  )
+chain = CanvasSync.default_provisioning_report_chain
+chain << {
+  job: CanvasSync::Jobs::SyncSimpleTableJob,
+  options: {
+    report_name: <report name>,
+    model: <model to sync>,
+    params: <hash with the require parameters the report needs to sync>
+  },
+}
+chain.process!
 ```
 
 ## Configuration
@@ -421,7 +334,6 @@ class CanvasSyncStarterWorker
         }
       }
     )
-    CanvasSync.invoke_next(job_chain)
   end
 
   def self.handle_canvas_sync_error(error, **options)

data/app/models/canvas_sync/sync_batch.rb

@@ -0,0 +1,5 @@
+module CanvasSync
+  class SyncBatch < ApplicationRecord
+    serialize :job_arguments, Array
+  end
+end

data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb

@@ -0,0 +1,11 @@
+class CreateCanvasSyncSyncBatches < CanvasSync::MiscHelper::MigrationClass
+  def change
+    create_table :canvas_sync_sync_batches do |t|
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.string :status
+
+      t.timestamps
+    end
+  end
+end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -64,12 +64,13 @@ module CanvasSync
         columns = columns.dup
 
         update_conditions = {
-          condition: condition_sql(klass, columns, import_args[:sync_start_time]),
+          condition: condition_sql(klass, columns),
           columns: columns
         }
         update_conditions[:conflict_target] = conflict_target if conflict_target
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
+
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
         klass.import(columns, rows, options)
       end
@@ -84,14 +85,10 @@ module CanvasSync
       # started_at = Time.now
       # run_the_users_sync!
       # changed = User.where("updated_at >= ?", started_at)
-      def self.condition_sql(klass, columns, report_start)
+      def self.condition_sql(klass, columns)
         columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
         excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
-        condition_sql = "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
-        if klass.column_names.include?("updated_at") && report_start
-          condition_sql += " AND #{klass.quoted_table_name}.updated_at < '#{report_start}'"
-        end
-        condition_sql
+        "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
       end
 
       def self.batch_size

data/lib/canvas_sync/job.rb

@@ -3,7 +3,7 @@ require "active_job"
 module CanvasSync
   # Inherit from this class to build a Job that will log to the canvas_sync_job_logs table
   class Job < ActiveJob::Base
-    attr_reader :job_chain, :job_log
+    attr_reader :job_log
 
     before_enqueue do |job|
       create_job_log(job)
@@ -15,12 +15,6 @@ module CanvasSync
       @job_log.started_at = Time.now
       @job_log.save
 
-      if job.arguments[0].is_a?(Hash) && job.arguments[0].include?(:jobs)
-        # @job_chain = JobChain.new(job.arguments[0])
-        @job_chain = job.arguments[0]
-        job.arguments[0] = @job_chain
-      end
-
       begin
         block.call
         @job_log.status = JobLog::SUCCESS_STATUS
@@ -28,11 +22,11 @@ module CanvasSync
         @job_log.exception = "#{e.class}: #{e.message}"
         @job_log.backtrace = e.backtrace.join('\n')
         @job_log.status = JobLog::ERROR_STATUS
-        if @job_chain&.dig(:global_options, :on_failure)&.present?
+        if batch_context&.[](:on_failure)&.present?
           begin
-            class_name, method = @job_chain[:global_options][:on_failure].split('.')
+            class_name, method = batch_context[:on_failure].split('.')
             klass = class_name.constantize
-            klass.send(method.to_sym, e, job_chain: @job_chain, job_log: @job_log)
+            klass.send(method.to_sym, e, batch_context: batch_context, job_log: @job_log)
           rescue => e2
             @job_log.backtrace += "\n\nError Occurred while handling an Error: #{e2.class}: #{e2.message}"
             @job_log.backtrace += "\n" + e2.backtrace.join('\n')