canvas_sync 0.12.0 → 0.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7795e4388557d88e9134d2c5ae6dee8770ba282f1dd72a10686aaa7974741c92
-  data.tar.gz: db4251351aabda748429adb17bdcb3c3dbc383d0dc2a7a112c324d0e8e273067
+  metadata.gz: b7038c74006653dd7e024f38b64df7e73862aab4f6249df28ef9f7ac342700bc
+  data.tar.gz: 7a485d873c8c76ef54d6bd7fe11bd98d697dd0d8a1954035396da74b414ec105
 SHA512:
-  metadata.gz: 3babc49ea2a583a81560aed52f08dcf7c4a706b74123fb1128b04f77030c575c3764feda1f5aab33c0e009b5c7d959cd2ebd7cdd79f566d9fdbad1f7a57d204d
-  data.tar.gz: '0048f0ebb889753af134a28a592a7b1ebee8f6b5c4d67c9ed816271fc4a14356a62ff8643b9bdfaf9a3e04046a95035768248790d56366b0db6f6a58701c2d4f'
+  metadata.gz: 0c1029ad4dc626d9924890d5b7cc9aaac2ae8e5e7b4350965e94e2545a14d29d650db42698bb16bd3cd97c00b6ff8571d4822ac98a9d808b3a7165e23bf449ff
+  data.tar.gz: 15a90b4b8df330db5fd7d4470d34487956de86a7e1e355ceb65bdfc7deb51174ab94b9feb013fc5b5fd431f6c89e58793f05ab23027d0f35fc7626936b0be7e5

data/README.md

@@ -223,7 +223,7 @@ users:
 
 ### API Sync
 Several models implement the `ApiSyncable` Concern. This is done in the Model Templates so as to be customizable and tweakable.
-Models that `include CanvasSync::ApiSyncable` should also call the `api_syncable` class method to configure the Synchronization.
+Models that `include CanvasSync::Concerns::ApiSyncable` should also call the `api_syncable` class method to configure the Synchronization.
 `api_syncable` takes two arguments and an optional block callback:
 ```ruby
 class CanvasSyncModel < ApplicationRecord

data/lib/canvas_sync.rb

@@ -1,33 +1,31 @@
 require "bearcat"
+
 require "canvas_sync/version"
 require "canvas_sync/engine"
+require "canvas_sync/class_callback_executor"
 require "canvas_sync/job"
+require "canvas_sync/job_chain"
 require "canvas_sync/sidekiq_job"
 require "canvas_sync/api_syncable"
+require "canvas_sync/record"
 require "canvas_sync/jobs/report_starter"
 require "canvas_sync/jobs/report_checker"
-require "canvas_sync/jobs/fork_gather"
 require "canvas_sync/jobs/report_processor_job"
-require "canvas_sync/jobs/sync_provisioning_report_job"
-require "canvas_sync/jobs/sync_simple_table_job.rb"
-require "canvas_sync/jobs/sync_assignments_job"
-require "canvas_sync/jobs/sync_submissions_job"
-require "canvas_sync/jobs/sync_assignment_groups_job"
-require "canvas_sync/jobs/sync_context_modules_job"
-require "canvas_sync/jobs/sync_context_module_items_job"
-require "canvas_sync/jobs/sync_terms_job"
-require "canvas_sync/jobs/sync_users_job"
-require "canvas_sync/jobs/sync_roles_job"
-require "canvas_sync/jobs/sync_admins_job"
 require "canvas_sync/config"
+
+Dir[File.dirname(__FILE__) +  "/canvas_sync/jobs/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/processors/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/importers/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/generators/*.rb"].each { |file| require file }
+Dir[File.dirname(__FILE__) +  "/canvas_sync/concerns/**/*.rb"].each { |file| require file }
 
 module CanvasSync
   SUPPORTED_MODELS = %w[
     users
+    pseudonyms
     courses
+    groups
+    group_memberships
     accounts
     terms
     enrollments
@@ -111,6 +109,8 @@ module CanvasSync
     #
     # @param job_chain [Hash] A chain of jobs to execute
     def invoke_next(job_chain, extra_options: {})
+      job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
+
       return if job_chain[:jobs].empty?
 
       # Make sure all job classes are serialized as strings
@@ -126,6 +126,8 @@ module CanvasSync
     end
 
     def fork(job_log, job_chain, keys: [])
+      job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
+
       duped_job_chain = Marshal.load(Marshal.dump(job_chain))
       duped_job_chain[:global_options][:fork_path] ||= []
       duped_job_chain[:global_options][:fork_keys] ||= []
@@ -144,6 +146,10 @@ module CanvasSync
       terms.each do |t|
         return scope.send(t) if scope.respond_to?(t)
       end
+      model = scope.try(:model) || scope
+      if model.try(:column_names)&.include?(:workflow_state)
+        return scope.where.not(workflow_state: %w[deleted])
+      end
       Rails.logger.warn("Could not filter Syncable Scope for model '#{scope.try(:model)&.name || scope.name}'")
       scope
     end
@@ -172,7 +178,7 @@ module CanvasSync
       global_options = {}
       global_options[:account_id] = account_id if account_id.present?
 
-      { jobs: jobs, global_options: global_options }
+      JobChain.new(jobs: jobs, global_options: global_options)
     end
 
     # Syncs terms, users/roles/admins if necessary, then the rest of the specified models.
@@ -193,7 +199,7 @@ module CanvasSync
 
       model_job_map = {
         terms: CanvasSync::Jobs::SyncTermsJob,
-        users: CanvasSync::Jobs::SyncUsersJob,
+        accounts: CanvasSync::Jobs::SyncAccountsJob,
         roles: CanvasSync::Jobs::SyncRolesJob,
         admins: CanvasSync::Jobs::SyncAdminsJob,
 
@@ -219,42 +225,91 @@ module CanvasSync
       models.unshift('terms') unless models.include?('terms')
       try_add_model_job.call('terms')
 
-      # Users, roles, and admins are synced before provisioning because they cannot be scoped to term
-      try_add_model_job.call('users') if term_scope.present?
+      # Accounts, users, roles, and admins are synced before provisioning because they cannot be scoped to term
+      try_add_model_job.call('accounts')
+
+      # These Models use the provisioning report, but are not term-scoped,
+      # so we sync them before to ensure work is not duplicated
+      if term_scope.present?
+        models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
+        jobs.concat(
+          generate_provisioning_jobs(first_provisioning_models, options)
+        )
+      end
+
       try_add_model_job.call('roles')
       try_add_model_job.call('admins')
-
       pre_provisioning_jobs = jobs
-      jobs = []
 
       ###############################
       # Post provisioning report jobs
       ###############################
 
+      jobs = []
       try_add_model_job.call('assignments')
       try_add_model_job.call('submissions')
       try_add_model_job.call('assignment_groups')
       try_add_model_job.call('context_modules')
       try_add_model_job.call('context_module_items')
-
       post_provisioning_jobs = jobs
 
-      jobs = pre_provisioning_jobs
-      if models.present?
-        provisioning_job = {
-          job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s,
-          options: { term_scope: term_scope, models: models },
-        }
-        provisioning_job[:options].merge!(options[:provisioning]) if options[:provisioning].present?
-        jobs += Array.wrap(provisioning_job)
-      end
-      jobs += post_provisioning_jobs
+      ###############################
+      # Main provisioning job and queueing
+      ###############################
+
+      jobs = [
+        *pre_provisioning_jobs,
+        *generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']),
+        *post_provisioning_jobs,
+      ]
 
       global_options = { legacy_support: legacy_support }
       global_options[:account_id] = account_id if account_id.present?
       global_options.merge!(options[:global]) if options[:global].present?
 
-      { jobs: jobs, global_options: global_options }
+      JobChain.new(jobs: jobs, global_options: global_options)
+    end
+
+    def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
+      dup_models = [ *model_list ]
+      unique_option_models = {}
+
+      filtered_models = only_split ? (only_split & model_list) : model_list
+      filtered_models.each do |m|
+        mopts = options_hash[m.to_sym] || options_hash[default_key]
+        unique_option_models[mopts] ||= []
+        unique_option_models[mopts] << m
+        dup_models.delete(m)
+      end
+
+      if dup_models.present?
+        mopts = options_hash[default_key]
+        unique_option_models[mopts] ||= []
+        unique_option_models[mopts].concat(dup_models)
+      end
+
+      unique_option_models
+    end
+
+    def generate_provisioning_jobs(model_list, options_hash, job_options: {}, only_split: nil, default_key: :provisioning)
+      # Group the model options as best we can.
+      # This is mainly for backwards compatibility, since 'users' was previously it's own job
+      unique_option_models = group_by_job_options(
+        model_list,
+        options_hash,
+        only_split: only_split,
+        default_key: default_key,
+      )
+
+      unique_option_models.map do |mopts, models|
+        opts = { models: models }
+        opts.merge!(job_options)
+        opts.merge!(mopts) if mopts.present?
+        {
+          job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s,
+          options: opts,
+        }
+      end
     end
 
     # Calls the canvas_sync_client in your app. If you have specified an account

data/lib/canvas_sync/api_syncable.rb

@@ -1,167 +1,9 @@
+# DEPRECATED - See CHANGELOG for 0.13.0
+# TODO: (0.14.0) Remove this module
 module CanvasSync::ApiSyncable
   extend ActiveSupport::Concern
-  NON_EXISTANT_ERRORS = [Faraday::Error::ResourceNotFound, Footrest::HttpError::NotFound]
 
-  class_methods do
-    def find_or_fetch(canvas_id, save: false, retries: 1)
-      inst = find_by(canvas_id: canvas_id)
-      return inst if inst.present?
-      inst = new(canvas_id: canvas_id)
-      api_response = inst.request_from_api(retries: retries)
-      inst.update_from_api_params(api_response)
-      inst.save! if save
-      inst
-    rescue *NON_EXISTANT_ERRORS
-      nil
-    end
-
-    def find_or_fetch!(*args)
-      inst = find_or_fetch(*args)
-      raise ActiveRecord::RecordNotFound unless inst.present?
-      inst
-    end
-
-    def create_or_update_from_api_params(api_params)
-      api_params = api_params.with_indifferent_access
-      inst = find_or_initialize_by(canvas_id: api_params[:id])
-      inst.update_from_api_params(api_params)
-      inst.save! if inst.changed?
-      inst
-    end
-
-    def api_sync_options=(opts)
-      @api_sync_options = opts
-    end
-
-    def api_sync_options
-      @api_sync_options || superclass.try(:api_sync_options)
-    end
-
-    # Define the model as being syncable via the Canvas API and configure sync options/parameters
-    # @param [Hash] map A hash of local_field => (:api_response_key | ->(api_response){ value })
-    # @param [->(bearcat?){ api_response }] fetch <description>
-    # @param [Hash] options <description>
-    # @option options [] :mark_deleted Hash to be merged | Symbol to invoke | ->(){ }
-    # @yield [api_response, [mapped_data]] Callback to merge data into a Model instance
-    def api_syncable(map, fetch, options={}, &blk)
-      default_options = {
-        mark_deleted: -> {
-          %i[workflow_state= status=].each do |sym|
-            next unless self.respond_to?(sym)
-            self.send(sym, 'deleted')
-            return
-          end
-        },
-        field_map: map,
-        fetch_from_api: fetch,
-        process_response: blk,
-      }
-      default_options.merge!(options)
-      self.api_sync_options = default_options.merge!(options)
-    end
-  end
-
-  # Call the API and Syncs this model.
-  # Calls the mark_deleted workflow if a 404 is received.
-  # @param [Number] retries Number of retries
-  # @return [Hash]  Response Hash from API
-  def sync_from_api(retries: 3)
-    api_response = request_from_api(retries: retries)
-    update_from_api_params!(api_response)
-    api_response
-  rescue *NON_EXISTANT_ERRORS
-    api_mark_deleted
-    save! if changed?
-    nil
-  end
-
-  # Fetch this model from the API and return the response
-  # @param [Number] retries Number of retries
-  # @return [Hash] Response Hash from API
-  def request_from_api(retries: 3)
-    api_call_with_retry(retries || 3) {
-      blk = api_sync_options[:fetch_from_api]
-      case blk.arity
-      when 1
-        self.instance_exec(canvas_sync_client, &blk)
-      else
-        self.instance_exec(&blk)
-      end
-    }
-  end
-
-  # Apply a response Hash from the API to this model's attributes, but do not save
-  # @param [Hash] api_params API-format Hash
-  # @return [self] self
-  def update_from_api_params(api_params)
-    options = self.api_sync_options
-    api_params = api_params.with_indifferent_access
-
-    map = options[:field_map]
-    mapped_params = {}
-    if map.present?
-      map.each do |local_name, remote_name|
-        if remote_name.respond_to?(:call)
-          mapped_params[local_name] = self.instance_exec(api_params, &remote_name)
-        elsif api_params.include?(remote_name)
-          mapped_params[local_name] = api_params[remote_name]
-          if remote_name == :id
-            current_value = send("#{local_name}")
-            raise "Mismatched Canvas ID" if current_value.present? && current_value != api_params[remote_name]
-          end
-        end
-      end
-    end
-
-    apply_block = options[:process_response]
-    if apply_block.present?
-      case apply_block.arity
-      when 1
-        self.instance_exec(api_params, &apply_block)
-      when 2
-        self.instance_exec(api_params, mapped_params, &apply_block)
-      end
-    else
-      mapped_params.each do |local_name, val|
-        send("#{local_name}=", val)
-      end
-    end
-    self
-  end
-
-  # Apply a response Hash from the API to this model's attributes, and save if changed?
-  # @param [Hash] api_params API-format Hash
-  # @return [self] self
-  def update_from_api_params!(api_params)
-    update_from_api_params(api_params)
-    save! if changed?
-    self
-  end
-
-  def api_sync_options
-    self.class.api_sync_options
-  end
-
-  private
-
-  def api_call_with_retry(retries=3)
-    tries ||= retries
-    yield
-  rescue Faraday::ConnectionFailed => e
-    raise e if (tries -= 1).zero?
-    sleep 5
-    retry
-  end
-
-  def api_mark_deleted
-    action = api_sync_options[:mark_deleted]
-    case action
-    when Hash
-      assign_attributes(action)
-    when Symbol
-      send(action)
-    when Proc
-      self.instance_exec(&action)
-    end
+  included do
+    raise "CanvasSync 0.13.0 includes breaking changes to ApiSyncable. See the CHANGELOG for upgrade steps."
   end
 end

data/lib/canvas_sync/class_callback_executor.rb

@@ -0,0 +1,35 @@
+module CanvasSync
+  # Helper/Hack class to allow calling ActiveSupport callbacks on a class instead of just on instances
+  class ClassCallbackExecutor
+    include ActiveSupport::Callbacks
+
+    attr_reader :callback_class
+    delegate :__callbacks, to: :callback_class
+    delegate_missing_to :callback_class
+
+    def initialize(cls, env)
+      @callback_class = cls
+      env.keys.each do |k|
+        define_singleton_method(k) do
+          env[k]
+        end
+      end
+    end
+
+    def clazz
+      callback_class
+    end
+
+    def self.run_callbacks(cls, callback, env={}, &blk)
+      new(cls, env).run_callbacks(callback, &blk)
+    end
+
+    def self.run_if_defined(cls, callback, *args, &blk)
+      if cls.respond_to?(:"_#{callback}_callbacks")
+        run_callbacks(cls, callback, *args, &blk)
+      else
+        blk.call
+      end
+    end
+  end
+end

data/lib/canvas_sync/concerns/account/ancestry.rb

@@ -0,0 +1,60 @@
+module CanvasSync::Concerns
+  module Account
+    # Add support for the ancestry Gem to Accounts
+    #
+    # Requires `ancestry` to be added to the Gemfile and a migration to execute these steps:
+    # add_column :accounts, :ancestry, :string
+    # add_index :accounts, :ancestry
+    #
+    # Handles syncing any Ancestry changes after CanvasSync syncs Accounts.
+    module Ancestry
+      extend ActiveSupport::Concern
+      include CanvasSync::Record
+
+      included do
+        has_ancestry
+        before_save :relink_ancestry, if: :canvas_parent_account_id_changed?
+        after_sync_import :ancestry_after_sync
+      end
+
+      class_methods do
+        def ancestry_after_sync
+          trails = {}
+          includes(:canvas_parent).find_each do |account|
+            parent = account.canvas_parent
+            trail = trails[parent.canvas_id] if parent.present?
+
+            if trail.present?
+              account.ancestry = trail
+              new_trail = "#{trail}/#{account.id.to_s}"
+            elsif parent.present?
+              account.parent = parent
+              new_trail = "#{account.ancestry}/#{account.id.to_s}"
+            else
+              account.parent = parent
+              new_trail = account.id.to_s
+            end
+
+            trails[account.canvas_id] = new_trail
+            account.save! if account.changed?
+          end
+        end
+      end
+
+      def relink_ancestry
+        self.parent = canvas_parent
+      end
+
+      def ensure_ancestry
+        return unless canvas_parent_account_id.present?
+        return if canvas_parent.present?
+
+        self.canvas_parent = Account.find_or_fetch(canvas_parent_account_id)
+        canvas_parent.save!
+        canvas_parent.ensure_ancestry
+        relink_ancestry
+        save! if changed?
+      end
+    end
+  end
+end