canvas_sync 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7795e4388557d88e9134d2c5ae6dee8770ba282f1dd72a10686aaa7974741c92
-  data.tar.gz: db4251351aabda748429adb17bdcb3c3dbc383d0dc2a7a112c324d0e8e273067
+  metadata.gz: 5c8386afbd924c0148b5c2158d23ea7acd5b0b85349da51312103e1ea9368c54
+  data.tar.gz: 051471b40b5a464b66df8ca7ae482874f2791c55604e28f7fe9ce54cb07f875a
 SHA512:
-  metadata.gz: 3babc49ea2a583a81560aed52f08dcf7c4a706b74123fb1128b04f77030c575c3764feda1f5aab33c0e009b5c7d959cd2ebd7cdd79f566d9fdbad1f7a57d204d
-  data.tar.gz: '0048f0ebb889753af134a28a592a7b1ebee8f6b5c4d67c9ed816271fc4a14356a62ff8643b9bdfaf9a3e04046a95035768248790d56366b0db6f6a58701c2d4f'
+  metadata.gz: '00742595c4797ba758ef98ab8b034545eb2193a2cae92dfc585e720c7cd9adef663ba0aadcae2c7085e19497ff74461c6d5553d49114f42dac520c8fe48ca448'
+  data.tar.gz: d8c7eb61e630bb65ea9aa2e86b86c03c2c1f87c151e127314c938f7d1b2438c90dfc5f5ddcc4cb74fe0b525bc4ef9c03dfb140d09fc10045b2f7f56b41524e43

data/README.md

@@ -223,7 +223,7 @@ users:
 
 ### API Sync
 Several models implement the `ApiSyncable` Concern. This is done in the Model Templates so as to be customizable and tweakable.
-Models that `include CanvasSync::ApiSyncable` should also call the `api_syncable` class method to configure the Synchronization.
+Models that `include CanvasSync::Concerns::ApiSyncable` should also call the `api_syncable` class method to configure the Synchronization.
 `api_syncable` takes two arguments and an optional block callback:
 ```ruby
 class CanvasSyncModel < ApplicationRecord

data/lib/canvas_sync.rb

@@ -1,28 +1,23 @@
 require "bearcat"
+
 require "canvas_sync/version"
 require "canvas_sync/engine"
+require "canvas_sync/class_callback_executor"
 require "canvas_sync/job"
+require "canvas_sync/job_chain"
 require "canvas_sync/sidekiq_job"
 require "canvas_sync/api_syncable"
+require "canvas_sync/record"
 require "canvas_sync/jobs/report_starter"
 require "canvas_sync/jobs/report_checker"
-require "canvas_sync/jobs/fork_gather"
 require "canvas_sync/jobs/report_processor_job"
-require "canvas_sync/jobs/sync_provisioning_report_job"
-require "canvas_sync/jobs/sync_simple_table_job.rb"
-require "canvas_sync/jobs/sync_assignments_job"
-require "canvas_sync/jobs/sync_submissions_job"
-require "canvas_sync/jobs/sync_assignment_groups_job"
-require "canvas_sync/jobs/sync_context_modules_job"
-require "canvas_sync/jobs/sync_context_module_items_job"
-require "canvas_sync/jobs/sync_terms_job"
-require "canvas_sync/jobs/sync_users_job"
-require "canvas_sync/jobs/sync_roles_job"
-require "canvas_sync/jobs/sync_admins_job"
 require "canvas_sync/config"
+
+Dir[File.dirname(__FILE__) +  "/canvas_sync/jobs/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/processors/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/importers/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/generators/*.rb"].each { |file| require file }
+Dir[File.dirname(__FILE__) +  "/canvas_sync/concerns/**/*.rb"].each { |file| require file }
 
 module CanvasSync
   SUPPORTED_MODELS = %w[
@@ -111,6 +106,8 @@ module CanvasSync
     #
     # @param job_chain [Hash] A chain of jobs to execute
     def invoke_next(job_chain, extra_options: {})
+      job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
+
       return if job_chain[:jobs].empty?
 
       # Make sure all job classes are serialized as strings
@@ -126,6 +123,8 @@ module CanvasSync
     end
 
     def fork(job_log, job_chain, keys: [])
+      job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
+
       duped_job_chain = Marshal.load(Marshal.dump(job_chain))
       duped_job_chain[:global_options][:fork_path] ||= []
       duped_job_chain[:global_options][:fork_keys] ||= []
@@ -172,7 +171,7 @@ module CanvasSync
       global_options = {}
       global_options[:account_id] = account_id if account_id.present?
 
-      { jobs: jobs, global_options: global_options }
+      JobChain.new(jobs: jobs, global_options: global_options)
     end
 
     # Syncs terms, users/roles/admins if necessary, then the rest of the specified models.
@@ -193,6 +192,7 @@ module CanvasSync
 
       model_job_map = {
         terms: CanvasSync::Jobs::SyncTermsJob,
+        accounts: CanvasSync::Jobs::SyncAccountsJob,
         users: CanvasSync::Jobs::SyncUsersJob,
         roles: CanvasSync::Jobs::SyncRolesJob,
         admins: CanvasSync::Jobs::SyncAdminsJob,
@@ -219,7 +219,8 @@ module CanvasSync
       models.unshift('terms') unless models.include?('terms')
       try_add_model_job.call('terms')
 
-      # Users, roles, and admins are synced before provisioning because they cannot be scoped to term
+      # Accounts, users, roles, and admins are synced before provisioning because they cannot be scoped to term
+      try_add_model_job.call('accounts')
       try_add_model_job.call('users') if term_scope.present?
       try_add_model_job.call('roles')
       try_add_model_job.call('admins')
@@ -254,7 +255,7 @@ module CanvasSync
       global_options[:account_id] = account_id if account_id.present?
       global_options.merge!(options[:global]) if options[:global].present?
 
-      { jobs: jobs, global_options: global_options }
+      JobChain.new(jobs: jobs, global_options: global_options)
     end
 
     # Calls the canvas_sync_client in your app. If you have specified an account

data/lib/canvas_sync/api_syncable.rb

@@ -1,167 +1,9 @@
+# DEPRECATED - See CHANGELOG for 0.13.0
+# TODO: (0.14.0) Remove this module
 module CanvasSync::ApiSyncable
   extend ActiveSupport::Concern
-  NON_EXISTANT_ERRORS = [Faraday::Error::ResourceNotFound, Footrest::HttpError::NotFound]
 
-  class_methods do
-    def find_or_fetch(canvas_id, save: false, retries: 1)
-      inst = find_by(canvas_id: canvas_id)
-      return inst if inst.present?
-      inst = new(canvas_id: canvas_id)
-      api_response = inst.request_from_api(retries: retries)
-      inst.update_from_api_params(api_response)
-      inst.save! if save
-      inst
-    rescue *NON_EXISTANT_ERRORS
-      nil
-    end
-
-    def find_or_fetch!(*args)
-      inst = find_or_fetch(*args)
-      raise ActiveRecord::RecordNotFound unless inst.present?
-      inst
-    end
-
-    def create_or_update_from_api_params(api_params)
-      api_params = api_params.with_indifferent_access
-      inst = find_or_initialize_by(canvas_id: api_params[:id])
-      inst.update_from_api_params(api_params)
-      inst.save! if inst.changed?
-      inst
-    end
-
-    def api_sync_options=(opts)
-      @api_sync_options = opts
-    end
-
-    def api_sync_options
-      @api_sync_options || superclass.try(:api_sync_options)
-    end
-
-    # Define the model as being syncable via the Canvas API and configure sync options/parameters
-    # @param [Hash] map A hash of local_field => (:api_response_key | ->(api_response){ value })
-    # @param [->(bearcat?){ api_response }] fetch <description>
-    # @param [Hash] options <description>
-    # @option options [] :mark_deleted Hash to be merged | Symbol to invoke | ->(){ }
-    # @yield [api_response, [mapped_data]] Callback to merge data into a Model instance
-    def api_syncable(map, fetch, options={}, &blk)
-      default_options = {
-        mark_deleted: -> {
-          %i[workflow_state= status=].each do |sym|
-            next unless self.respond_to?(sym)
-            self.send(sym, 'deleted')
-            return
-          end
-        },
-        field_map: map,
-        fetch_from_api: fetch,
-        process_response: blk,
-      }
-      default_options.merge!(options)
-      self.api_sync_options = default_options.merge!(options)
-    end
-  end
-
-  # Call the API and Syncs this model.
-  # Calls the mark_deleted workflow if a 404 is received.
-  # @param [Number] retries Number of retries
-  # @return [Hash]  Response Hash from API
-  def sync_from_api(retries: 3)
-    api_response = request_from_api(retries: retries)
-    update_from_api_params!(api_response)
-    api_response
-  rescue *NON_EXISTANT_ERRORS
-    api_mark_deleted
-    save! if changed?
-    nil
-  end
-
-  # Fetch this model from the API and return the response
-  # @param [Number] retries Number of retries
-  # @return [Hash] Response Hash from API
-  def request_from_api(retries: 3)
-    api_call_with_retry(retries || 3) {
-      blk = api_sync_options[:fetch_from_api]
-      case blk.arity
-      when 1
-        self.instance_exec(canvas_sync_client, &blk)
-      else
-        self.instance_exec(&blk)
-      end
-    }
-  end
-
-  # Apply a response Hash from the API to this model's attributes, but do not save
-  # @param [Hash] api_params API-format Hash
-  # @return [self] self
-  def update_from_api_params(api_params)
-    options = self.api_sync_options
-    api_params = api_params.with_indifferent_access
-
-    map = options[:field_map]
-    mapped_params = {}
-    if map.present?
-      map.each do |local_name, remote_name|
-        if remote_name.respond_to?(:call)
-          mapped_params[local_name] = self.instance_exec(api_params, &remote_name)
-        elsif api_params.include?(remote_name)
-          mapped_params[local_name] = api_params[remote_name]
-          if remote_name == :id
-            current_value = send("#{local_name}")
-            raise "Mismatched Canvas ID" if current_value.present? && current_value != api_params[remote_name]
-          end
-        end
-      end
-    end
-
-    apply_block = options[:process_response]
-    if apply_block.present?
-      case apply_block.arity
-      when 1
-        self.instance_exec(api_params, &apply_block)
-      when 2
-        self.instance_exec(api_params, mapped_params, &apply_block)
-      end
-    else
-      mapped_params.each do |local_name, val|
-        send("#{local_name}=", val)
-      end
-    end
-    self
-  end
-
-  # Apply a response Hash from the API to this model's attributes, and save if changed?
-  # @param [Hash] api_params API-format Hash
-  # @return [self] self
-  def update_from_api_params!(api_params)
-    update_from_api_params(api_params)
-    save! if changed?
-    self
-  end
-
-  def api_sync_options
-    self.class.api_sync_options
-  end
-
-  private
-
-  def api_call_with_retry(retries=3)
-    tries ||= retries
-    yield
-  rescue Faraday::ConnectionFailed => e
-    raise e if (tries -= 1).zero?
-    sleep 5
-    retry
-  end
-
-  def api_mark_deleted
-    action = api_sync_options[:mark_deleted]
-    case action
-    when Hash
-      assign_attributes(action)
-    when Symbol
-      send(action)
-    when Proc
-      self.instance_exec(&action)
-    end
+  included do
+    raise "CanvasSync 0.13.0 includes breaking changes to ApiSyncable. See the CHANGELOG for upgrade steps."
   end
 end

data/lib/canvas_sync/class_callback_executor.rb

@@ -0,0 +1,35 @@
+module CanvasSync
+  # Helper/Hack class to allow calling ActiveSupport callbacks on a class instead of just on instances
+  class ClassCallbackExecutor
+    include ActiveSupport::Callbacks
+
+    attr_reader :callback_class
+    delegate :__callbacks, to: :callback_class
+    delegate_missing_to :callback_class
+
+    def initialize(cls, env)
+      @callback_class = cls
+      env.keys.each do |k|
+        define_singleton_method(k) do
+          env[k]
+        end
+      end
+    end
+
+    def clazz
+      callback_class
+    end
+
+    def self.run_callbacks(cls, callback, env={}, &blk)
+      new(cls, env).run_callbacks(callback, &blk)
+    end
+
+    def self.run_if_defined(cls, callback, *args, &blk)
+      if cls.respond_to?(:"_#{callback}_callbacks")
+        run_callbacks(cls, callback, *args, &blk)
+      else
+        blk.call
+      end
+    end
+  end
+end

data/lib/canvas_sync/concerns/account/ancestry.rb

@@ -0,0 +1,60 @@
+module CanvasSync::Concerns
+  module Account
+    # Add support for the ancestry Gem to Accounts
+    #
+    # Requires `ancestry` to be added to the Gemfile and a migration to execute these steps:
+    # add_column :accounts, :ancestry, :string
+    # add_index :accounts, :ancestry
+    #
+    # Handles syncing any Ancestry changes after CanvasSync syncs Accounts.
+    module Ancestry
+      extend ActiveSupport::Concern
+      include CanvasSync::Record
+
+      included do
+        has_ancestry
+        before_save :relink_ancestry, if: :canvas_parent_account_id_changed?
+        after_sync_import :ancestry_after_sync
+      end
+
+      class_methods do
+        def ancestry_after_sync
+          trails = {}
+          includes(:canvas_parent).find_each do |account|
+            parent = account.canvas_parent
+            trail = trails[parent.canvas_id] if parent.present?
+
+            if trail.present?
+              account.ancestry = trail
+              new_trail = "#{trail}/#{account.id.to_s}"
+            elsif parent.present?
+              account.parent = parent
+              new_trail = "#{account.ancestry}/#{account.id.to_s}"
+            else
+              account.parent = parent
+              new_trail = account.id.to_s
+            end
+
+            trails[account.canvas_id] = new_trail
+            account.save! if account.changed?
+          end
+        end
+      end
+
+      def relink_ancestry
+        self.parent = canvas_parent
+      end
+
+      def ensure_ancestry
+        return unless canvas_parent_account_id.present?
+        return if canvas_parent.present?
+
+        self.canvas_parent = Account.find_or_fetch(canvas_parent_account_id)
+        canvas_parent.save!
+        canvas_parent.ensure_ancestry
+        relink_ancestry
+        save! if changed?
+      end
+    end
+  end
+end

data/lib/canvas_sync/concerns/api_syncable.rb

@@ -0,0 +1,189 @@
+module CanvasSync::Concerns
+  module ApiSyncable
+    extend ActiveSupport::Concern
+    NON_EXISTANT_ERRORS = [Faraday::Error::ResourceNotFound, Footrest::HttpError::NotFound]
+
+    class_methods do
+      def find_or_fetch(canvas_id, save: false, retries: 1, **kwargs)
+        inst = find_or_initialize_by(canvas_id: canvas_id)
+        return inst if inst.persisted?
+
+        api_response = inst.request_from_api(retries: retries, **kwargs)
+        api_sync_race_create!(inst, save: save) do |inst2|
+          inst2.assign_from_api_params(api_response, **kwargs)
+        end
+      rescue *NON_EXISTANT_ERRORS
+        nil
+      end
+
+      def find_or_fetch!(*args)
+        inst = find_or_fetch(*args)
+        raise ActiveRecord::RecordNotFound unless inst.present?
+        inst
+      end
+
+      def create_or_update_from_api_params(api_params)
+        api_params = api_params.with_indifferent_access
+        api_sync_race_create!(api_params[:id]) do |inst|
+          inst.assign_from_api_params(api_params)
+        end
+      end
+
+      def api_sync_options=(opts)
+        @api_sync_options = opts
+      end
+
+      def api_sync_options
+        @api_sync_options || superclass.try(:api_sync_options)
+      end
+
+      # Define the model as being syncable via the Canvas API and configure sync options/parameters
+      # @param [Hash] map A hash of local_field => (:api_response_key | ->(api_response){ value })
+      # @param [->(bearcat?){ api_response }] fetch <description>
+      # @param [Hash] options <description>
+      # @option options [] :mark_deleted Hash to be merged | Symbol to invoke | ->(){ }
+      # @yield [api_response, [mapped_data]] Callback to merge data into a Model instance
+      def api_syncable(map, fetch, options={}, &blk)
+        default_options = {
+          mark_deleted: -> {
+            %i[workflow_state= status=].each do |sym|
+              next unless self.respond_to?(sym)
+              self.send(sym, 'deleted')
+              return
+            end
+          },
+          field_map: map,
+          fetch_from_api: fetch,
+          process_response: blk,
+        }
+        default_options.merge!(options)
+        self.api_sync_options = default_options.merge!(options)
+      end
+
+      private
+
+      def api_sync_race_create!(inst, save: true)
+        inst = find_or_initialize_by(canvas_id: inst) unless inst.is_a?(self)
+        yield inst
+        inst.save! if save && inst.changed?
+        inst
+      rescue ActiveRecord::RecordNotUnique
+        inst = find_by(canvas_id: inst.canvas_id)
+        yield inst
+        inst.save! if save && inst.changed?
+        inst
+      end
+    end
+
+    # Call the API and Syncs this model.
+    # Calls the mark_deleted workflow if a 404 is received.
+    # @param [Number] retries Number of retries
+    # @return [Hash]  Response Hash from API
+    def sync_from_api(retries: 3, **kwargs)
+      api_response = request_from_api(retries: retries, **kwargs)
+      update_from_api_params!(api_response, **kwargs)
+      api_response
+    rescue *NON_EXISTANT_ERRORS
+      api_mark_deleted
+      save! if changed?
+      nil
+    end
+
+    # Fetch this model from the API and return the response
+    # @param [Number] retries Number of retries
+    # @return [Hash] Response Hash from API
+    def request_from_api(retries: 3, **kwargs)
+      api_call_with_retry(retries || 3) {
+        blk = api_sync_options[:fetch_from_api]
+        case blk.arity
+        when 1
+          self.instance_exec(canvas_sync_client, &blk)
+        else
+          self.instance_exec(&blk)
+        end
+      }
+    end
+
+    # Apply a response Hash from the API to this model's attributes, but do not save
+    # @param [Hash] api_params API-format Hash
+    # @return [self] self
+    def assign_from_api_params(api_params, **kwargs)
+      options = self.api_sync_options
+      api_params = api_params.with_indifferent_access
+
+      map = options[:field_map]
+      mapped_params = {}
+      if map.present?
+        map.each do |local_name, remote_name|
+          if remote_name.respond_to?(:call)
+            mapped_params[local_name] = self.instance_exec(api_params, &remote_name)
+          elsif api_params.include?(remote_name)
+            mapped_params[local_name] = api_params[remote_name]
+            if remote_name == :id
+              current_value = send("#{local_name}")
+              raise "Mismatched Canvas ID" if current_value.present? && current_value != api_params[remote_name]
+            end
+          end
+        end
+      end
+
+      apply_block = options[:process_response]
+      if apply_block.present?
+        case apply_block.arity
+        when 1
+          self.instance_exec(api_params, &apply_block)
+        when 2
+          self.instance_exec(api_params, mapped_params, &apply_block)
+        end
+      else
+        mapped_params.each do |local_name, val|
+          send("#{local_name}=", val)
+        end
+      end
+      self
+    end
+
+    # Apply a response Hash from the API to this model's attributes and save if changed?
+    # @param [Hash] api_params API-format Hash
+    # @return [self] self
+    def update_from_api_params(api_params, **kwargs)
+      assign_from_api_params(*args)
+      save if changed?
+    end
+
+    # Apply a response Hash from the API to this model's attributes, and save! if changed?
+    # @param [Hash] api_params API-format Hash
+    # @return [self] self
+    def update_from_api_params!(*args)
+      assign_from_api_params(*args)
+      save! if changed?
+    end
+
+    def api_sync_options
+      self.class.api_sync_options
+    end
+
+    private
+
+    def api_call_with_retry(retries=3)
+      tries ||= retries
+      yield
+    rescue Faraday::ConnectionFailed => e
+      raise e if (tries -= 1).zero?
+      sleep 5
+      retry
+    end
+
+    def api_mark_deleted
+      action = api_sync_options[:mark_deleted]
+      case action
+      when Hash
+        assign_attributes(action)
+      when Symbol
+        send(action)
+      when Proc
+        self.instance_exec(&action)
+      end
+    end
+  end
+end