canvas_sync 0.16.4 → 0.21.0

data/lib/canvas_sync/record.rb

@@ -1,9 +1,112 @@
+
+require_relative 'concerns/sync_mapping'
+
 module CanvasSync
   module Record
     extend ActiveSupport::Concern
+    include CanvasSync::Concerns::SyncMapping
+
+    def self.registered_features
+      @canvas_sync_features ||= {}
+    end
+
+    def self.define_feature(modul, feature_ident = nil, models: nil, default: false, &blk)
+      feature_ident = modul.name.demodulize.underscore if feature_ident.nil?
+      if models == nil
+        pmod = _get_module_parent(modul)
+        models = pmod.name.demodulize if pmod && _get_module_parent(pmod) == CanvasSync::Concerns
+      end
+
+      @canvas_sync_features ||= {}
+      @canvas_sync_features[feature_ident] = {
+        name: feature_ident,
+        module: modul,
+        default: default,
+        models: Array.wrap(models).map(&:to_s),
+        setup: blk || ->{ include modul },
+      }
+    end
+
+    def self._get_module_parent(modul)
+      modul.respond_to?(:parent) ? modul.parent : modul.module_parent
+    end
+
+    class_methods do
+      def applicable_canvas_sync_features
+        @applicable_canvas_sync_features ||= begin
+          relev_feats = {}
+          CanvasSync::Record.registered_features.each do |k, cfg|
+            next if cfg[:models].present? && !cfg[:models].include?(self.name.demodulize)
+            relev_feats[k] = cfg
+          end
+          relev_feats
+        end
+      end
+
+      def canvas_sync_feature(feature, config = nil)
+        feat = applicable_canvas_sync_features[feature]
+
+        if !feat
+          raise "Unknown feature :#{feature} for model #{self}"
+        end
+
+        setup = feat[:setup]
+        if setup.arity != 0
+          instance_exec(config, &setup)
+        elsif config.present?
+          raise ArgumentError, "Feature #{feature} does not accept configuration"
+        else
+          instance_exec(&setup)
+        end
+      end
+
+      def canvas_sync_features(*args, skip: [], **kwargs)
+        features = {}
+
+        include_defaults = args.length == 0 || args.include?(:defaults)
+        if include_defaults
+          applicable_canvas_sync_features.each do |key, cfg|
+            next if skip.include?(key)
+
+            case cfg[:default]
+            when true
+              features[key] = {}
+            when Proc
+              features[key] = {} if instance_exec(&cfg[:default])
+            end
+          end
+        end
+
+        args.each do |a|
+          next if a == :defaults
+          features[a] ||= {}
+        end
+
+        kwargs.each do |key, opts|
+          if opts == false
+            features.delete(key)
+          elsif opts == true
+            features[key] ||= {}
+          else
+            features[key] = opts
+          end
+        end
+
+        features.each do |key, opts|
+          canvas_sync_feature(key, opts)
+        end
+      end
+    end
 
     included do
       define_model_callbacks :sync_import
+      define_model_callbacks :sync_batch_import
+
+      col_names = column_names rescue []
+
+      if col_names.include?("canvas_sync_batch_id")
+        belongs_to :canvas_sync_batch, foreign_key: :canvas_sync_batch_id, class_name: "CanvasSync::SyncBatch", optional: true
+      end
     end
   end
 end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.16.4".freeze
+  VERSION = "0.21.0".freeze
 end

data/lib/canvas_sync.rb

@@ -2,17 +2,21 @@ require "bearcat"
 
 require "canvas_sync/version"
 require "canvas_sync/engine"
+require "canvas_sync/misc_helper"
 require "canvas_sync/class_callback_executor"
 require "canvas_sync/job"
-require "canvas_sync/job_chain"
 require "canvas_sync/sidekiq_job"
 require "canvas_sync/api_syncable"
 require "canvas_sync/record"
+require "canvas_sync/live_events"
 require "canvas_sync/jobs/report_starter"
 require "canvas_sync/jobs/report_checker"
 require "canvas_sync/jobs/report_processor_job"
+require "canvas_sync/batch_processor"
 require "canvas_sync/config"
 
+require "canvas_sync/job_batches/batch"
+
 Dir[File.dirname(__FILE__) +  "/canvas_sync/jobs/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/processors/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/importers/*.rb"].each { |file| require file }
@@ -38,6 +42,38 @@ module CanvasSync
     context_modules
     context_module_items
     xlist
+    user_observers
+    grading_periods
+    grading_period_groups
+    content_migrations
+    learning_outcomes
+    learning_outcome_results
+    course_nicknames
+    rubrics
+    rubric_associations
+    rubric_assessments
+  ].freeze
+
+  SUPPORTED_TERM_SCOPE_MODELS = %w[
+    assignments
+    submissions
+    assignment_groups
+    context_modules
+    context_module_items
+    rubrics
+    rubric_associations
+    rubric_assessments
+  ].freeze
+
+  DEFAULT_TERM_SCOPE_MODELS = %w[
+    assignments
+    submissions
+    assignment_groups
+    context_modules
+    context_module_items
+    rubrics
+    rubric_associations
+    rubric_assessments
   ].freeze
 
   SUPPORTED_LIVE_EVENTS = %w[
@@ -57,6 +93,10 @@ module CanvasSync
     graded_submissions
   ].freeze
 
+  JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::SyncTermsJob, :sub_jobs)
+  JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::TermBatchesJob, :sub_jobs)
+  JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::BeginSyncChainJob, 0)
+
   class << self
     # Runs a standard provisioning sync job with no extra report types.
     # Terms will be synced first using the API. If you are syncing users/roles/admins
@@ -72,72 +112,9 @@ module CanvasSync
     #   and inserts it into the database. If an array of model names is provided then only those models will use legacy support.
     # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
     #   canvas_sync_client methods require an account ID.
-    def provisioning_sync(models, term_scope: nil, legacy_support: false, account_id: nil)
+    def provisioning_sync(models, **kwargs)
       validate_models!(models)
-      invoke_next(default_provisioning_report_chain(models, term_scope, legacy_support, account_id))
-    end
-
-    # Runs a report different from provisioning sync job with no extra report types.
-    #
-    # @param reports_mapping [Array<Hash>] An Array of hash that list the model and params with their report you
-    #   want to import:
-    #   [{model: 'submissions', report_name: 'my_report_name_csv', params: { "parameters[include_deleted]" => true } }, ...]
-    # @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model.
-    #   The provisioning report will be run for each of the terms contained in that scope.
-    # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
-    #   canvas_sync_client methods require an account ID.
-    def simple_report_sync(reports_mapping, term_scope: nil, account_id: nil)
-      invoke_next(simple_report_chain(reports_mapping, term_scope, account_id))
-    end
-
-    # Runs a chain of ordered jobs
-    #
-    # See the README for usage and examples
-    #
-    # @param job_chain [Hash]
-    def process_jobs(job_chain)
-      invoke_next(job_chain)
-    end
-
-    def duplicate_chain(job_chain)
-      Marshal.load(Marshal.dump(job_chain))
-    end
-
-    # Invokes the next job in a chain of jobs.
-    #
-    # This should typically be called automatically by the gem where necessary.
-    #
-    # @param job_chain [Hash] A chain of jobs to execute
-    def invoke_next(job_chain, extra_options: {})
-      job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
-
-      return if job_chain[:jobs].empty?
-
-      # Make sure all job classes are serialized as strings
-      job_chain[:jobs].each { |job| job[:job] = job[:job].to_s }
-
-      duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-      jobs = duped_job_chain[:jobs]
-      next_job = jobs.shift
-      next_job_class = next_job[:job].constantize
-      next_options = next_job[:options] || {}
-      next_options.merge!(extra_options)
-      next_job_class.perform_later(duped_job_chain, next_options)
-    end
-
-    def fork(job_log, job_chain, keys: [])
-      job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
-
-      duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-      duped_job_chain[:global_options][:fork_path] ||= []
-      duped_job_chain[:global_options][:fork_keys] ||= []
-      duped_job_chain[:global_options][:fork_path] << job_log.job_id
-      duped_job_chain[:global_options][:fork_keys] << keys.map(&:to_s)
-      duped_job_chain[:global_options][:on_failure] ||= 'CanvasSync::Jobs::ForkGather.handle_branch_error'
-      sub_items = yield duped_job_chain
-      sub_count = sub_items.respond_to?(:count) ? sub_items.count : sub_items
-      job_log.fork_count = sub_count
-      sub_items
+      default_provisioning_report_chain(models, **kwargs).process!
     end
 
     # Given a Model or Relation, scope it down to items that should be synced
@@ -154,33 +131,6 @@ module CanvasSync
       scope
     end
 
-    # Syn any report to an specific set of models
-    #
-    # @param reports_mapping [Array<Hash>] List of reports with their specific model and params
-    # @param term_scope [String]
-    # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
-    #   canvas_sync_client methods require an account ID.
-    def simple_report_chain(reports_mapping, term_scope=nil, account_id=nil)
-      jobs = reports_mapping.map do |report|
-        {
-          job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
-          options: {
-            report_name: report[:report_name],
-            model: report[:model],
-            mapping: report[:model],
-            klass: report[:model].singularize.capitalize.to_s,
-            term_scope: term_scope,
-            params: report[:params]
-          }
-        }
-      end
-
-      global_options = {}
-      global_options[:account_id] = account_id if account_id.present?
-
-      JobChain.new(jobs: jobs, global_options: global_options)
-    end
-
     # Syncs terms, users/roles/admins if necessary, then the rest of the specified models.
     #
     # @param models [Array<String>]
@@ -191,7 +141,13 @@ module CanvasSync
     # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
     #   canvas_sync_client methods require an account ID.
     # @return [Hash]
-    def default_provisioning_report_chain(models, term_scope=nil, legacy_support=false, account_id=nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
+    def default_provisioning_report_chain(
+      models,
+      term_scope: nil,
+      term_scoped_models: DEFAULT_TERM_SCOPE_MODELS,
+      options: {},
+      **kwargs
+    ) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
       return unless models.present?
       models.map! &:to_s
       term_scope = term_scope.to_s if term_scope
@@ -208,66 +164,92 @@ module CanvasSync
         assignment_groups: CanvasSync::Jobs::SyncAssignmentGroupsJob,
         context_modules: CanvasSync::Jobs::SyncContextModulesJob,
         context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
+        content_migrations: CanvasSync::Jobs::SyncContentMigrationsJob,
+        rubrics: CanvasSync::Jobs::SyncRubricsJob,
+        rubric_associations: CanvasSync::Jobs::SyncRubricAssociationsJob,
+        rubric_assessments: CanvasSync::Jobs::SyncRubricAssessmentsJob,
       }.with_indifferent_access
 
-      jobs = []
+      root_chain = base_canvas_sync_chain(**kwargs, globals: options[:global] || kwargs[:globals])
+      concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
+      root_chain << concurrent_root_chain
+      current_chain = concurrent_root_chain
+
       try_add_model_job = ->(model) {
         return unless models.include?(model)
-        jobs.push(job: model_job_map[model].to_s, options: options[model.to_sym] || {})
+        current_chain << { job: model_job_map[model].to_s, options: options[model.to_sym] || {} }
         models -= [model]
       }
 
       ##############################
-      # Pre provisioning report jobs
+      # General provisioning jobs (not term-scoped)
       ##############################
 
-      # Always sync Terms first
-      models.unshift('terms') unless models.include?('terms')
-      try_add_model_job.call('terms')
-
-      # Accounts, users, roles, and admins are synced before provisioning because they cannot be scoped to term
+      # Accounts, users, roles, and admins cannot be scoped to term
       try_add_model_job.call('accounts')
 
       # These Models use the provisioning report, but are not term-scoped,
-      # so we sync them before to ensure work is not duplicated
+      # so we sync them outside of the term scoping to ensure work is not duplicated
       if term_scope.present?
-        models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
-        jobs.concat(
-          generate_provisioning_jobs(first_provisioning_models, options)
-        )
+        models -= (first_provisioning_models = models & ['users', 'pseudonyms', 'user_observers', 'grading_periods', 'grading_period_groups'])
+        current_chain.insert(generate_provisioning_jobs(first_provisioning_models, options))
       end
 
       try_add_model_job.call('roles')
       try_add_model_job.call('admins')
-      pre_provisioning_jobs = jobs
+
+      (SUPPORTED_TERM_SCOPE_MODELS - term_scoped_models).each do |mdl|
+        try_add_model_job.call(mdl)
+      end
 
       ###############################
-      # Post provisioning report jobs
+      # Per-term provisioning jobs
       ###############################
 
-      jobs = []
-      try_add_model_job.call('assignments')
-      try_add_model_job.call('submissions')
-      try_add_model_job.call('assignment_groups')
-      try_add_model_job.call('context_modules')
-      try_add_model_job.call('context_module_items')
-      post_provisioning_jobs = jobs
+      term_parent_chain = current_chain
+
+      per_term_chain = JobBatches::ChainBuilder.build(model_job_map[:terms], term_scope: term_scope)
+      current_chain = per_term_chain
+
+      term_scoped_models.each do |mdl|
+        try_add_model_job.call(mdl)
+      end
+
+      current_chain.insert(
+        generate_provisioning_jobs(models - ['terms'], options)
+      )
+
+      # Skip syncing terms if not required
+      if !current_chain.empty? || (models & ['terms']).present?
+        term_parent_chain << per_term_chain
+      end
 
       ###############################
-      # Main provisioning job and queueing
+      # Wrap it all up
       ###############################
 
-      jobs = [
-        *pre_provisioning_jobs,
-        *generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']),
-        *post_provisioning_jobs,
-      ]
+      root_chain
+    end
 
-      global_options = { legacy_support: legacy_support }
+    def base_canvas_sync_chain(
+      legacy_support: false, # Import records 1 by 1 instead of with bulk upserts
+      account_id: nil, # legacy/non PandaPal apps
+      updated_after: nil,
+      full_sync_every: nil,
+      batch_genre: nil,
+      globals: {},
+      &blk
+    )
+      global_options = {
+        legacy_support: legacy_support,
+        updated_after: updated_after,
+        full_sync_every: full_sync_every,
+        batch_genre: batch_genre,
+      }
       global_options[:account_id] = account_id if account_id.present?
-      global_options.merge!(options[:global]) if options[:global].present?
+      global_options.merge!(globals) if globals
 
-      JobChain.new(jobs: jobs, global_options: global_options)
+      JobBatches::ChainBuilder.build(CanvasSync::Jobs::BeginSyncChainJob, [], global_options, &blk)
     end
 
     def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
@@ -352,5 +334,22 @@ module CanvasSync
       return if invalid.empty?
       raise "Invalid live event(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_LIVE_EVENTS.join(', ')} are supported."
     end
+
+    def logger
+      return @logger if defined? @logger
+      @logger = Logger.new(STDOUT)
+      @logger.level = Logger::DEBUG
+      @logger
+    end
+
+    def redis(*args, &blk)
+      JobBatches::Batch.redis(*args, &blk)
+    end
+
+    def redis_prefix
+      pfx = "cs"
+      pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
+      pfx
+    end
   end
 end