canvas_sync 0.16.4 → 0.21.0

data/config/initializers/apartment.rb

@@ -5,6 +5,15 @@
 #   * https://github.com/influitive/apartment/issues/508
 
 Rails.application.config.after_initialize do
+  next unless defined?(Apartment)
+
+  # Newer versions of Apartment already solves this issue (and in a better way)
+  begin
+    require('apartment/version')
+    next if Gem::Version.new(Apartment::VERSION) >= Gem::Version.new('2.8.1')
+  rescue LoadError
+  end
+
   begin
     Rails.application.eager_load!
     ActiveRecord::Base.descendants.each do |model|
@@ -14,7 +23,7 @@ Rails.application.config.after_initialize do
         model.sequence_name = seq_name
       end
     end
-  rescue PG::ConnectionBad, ActiveRecord::NoDatabaseError
+  rescue PG::ConnectionBad, ActiveRecord::NoDatabaseError, ActiveRecord::ConnectionNotEstablished
     # no-op if there is no database setup
   end
 end

data/config/routes.rb

@@ -0,0 +1,7 @@
+CanvasSync::Engine.routes.draw do
+  namespace "api" do
+    namespace "v1" do
+      post '/live_event' => 'live_events#process_event'
+    end
+  end
+end

data/db/migrate/20170915210836_create_canvas_sync_job_log.rb

@@ -1,35 +1,16 @@
-if Rails.version.to_f >= 5.0
-  class CreateCanvasSyncJobLog < ActiveRecord::Migration[Rails.version.to_f]
-    def change
-      create_table :canvas_sync_job_logs do |t|
-        t.datetime :started_at
-        t.datetime :completed_at
-        t.string :exception
-        t.text :backtrace
-        t.string :job_class
-        t.string :status
-        t.text :metadata
-        t.text :job_arguments
+class CreateCanvasSyncJobLog < CanvasSync::MiscHelper::MigrationClass
+  def change
+    create_table :canvas_sync_job_logs do |t|
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.string :exception
+      t.text :backtrace
+      t.string :job_class
+      t.string :status
+      t.text :metadata
+      t.text :job_arguments
 
-        t.timestamps
-      end
-    end
-  end
-else
-  class CreateCanvasSyncJobLog < ActiveRecord::Migration
-    def change
-      create_table :canvas_sync_job_logs do |t|
-        t.datetime :started_at
-        t.datetime :completed_at
-        t.string :exception
-        t.text :backtrace
-        t.string :job_class
-        t.string :status
-        t.text :metadata
-        t.text :job_arguments
-
-        t.timestamps
-      end
+      t.timestamps
     end
   end
 end

data/db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb

@@ -1,15 +1,6 @@
-if Rails.version.to_f >= 5.0
-  class AddJobIdToCanvasSyncJobLogs < ActiveRecord::Migration[Rails.version.to_f]
-    def change
-      add_column :canvas_sync_job_logs, :job_id, :string
-      add_index :canvas_sync_job_logs, :job_id
-    end
-  end
-else
-  class AddJobIdToCanvasSyncJobLogs < ActiveRecord::Migration
-    def change
-      add_column :canvas_sync_job_logs, :job_id, :string
-      add_index :canvas_sync_job_logs, :job_id
-    end
+class AddJobIdToCanvasSyncJobLogs < CanvasSync::MiscHelper::MigrationClass
+  def change
+    add_column :canvas_sync_job_logs, :job_id, :string
+    add_index :canvas_sync_job_logs, :job_id
   end
 end

data/db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb

@@ -1,13 +1,5 @@
-if Rails.version.to_f >= 5.0
-  class AddForkCountToCanvasSyncJobLogs < ActiveRecord::Migration[Rails.version.to_f]
-    def change
-      add_column :canvas_sync_job_logs, :fork_count, :integer
-    end
-  end
-else
-  class AddForkCountToCanvasSyncJobLogs < ActiveRecord::Migration
-    def change
-      add_column :canvas_sync_job_logs, :fork_count, :integer
-    end
+class AddForkCountToCanvasSyncJobLogs < CanvasSync::MiscHelper::MigrationClass
+  def change
+    add_column :canvas_sync_job_logs, :fork_count, :integer
   end
 end

data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb

@@ -0,0 +1,11 @@
+class CreateCanvasSyncSyncBatches < CanvasSync::MiscHelper::MigrationClass
+  def change
+    create_table :canvas_sync_sync_batches do |t|
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.string :status
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20201030210836_add_full_sync_to_canvas_sync_sync_batch.rb

@@ -0,0 +1,7 @@
+class AddFullSyncToCanvasSyncSyncBatch < CanvasSync::MiscHelper::MigrationClass
+  def change
+    add_column :canvas_sync_sync_batches, :full_sync, :boolean, default: false
+    add_column :canvas_sync_sync_batches, :batch_genre, :string
+    add_column :canvas_sync_sync_batches, :batch_bid, :string
+  end
+end

data/lib/canvas_sync/batch_processor.rb

@@ -0,0 +1,41 @@
+module CanvasSync
+  # An array that "processes" after so many items are added.
+  #
+  # Example Usage:
+  #   batches = CanvasSync::BatchProcessor.new(of: 1000) do |batch|
+  #     # Process the batch somehow
+  #   end
+  #   enumerator_of_some_kind.each { |item| batches << item }
+  #   batches.flush
+  class BatchProcessor
+    attr_reader :batch_size
+
+    def initialize(of: 1000, &blk)
+      @batch_size = of
+      @block = blk
+      @current_batch = []
+    end
+
+    def <<(item)
+      @current_batch << item
+      process_batch if @current_batch.count >= batch_size
+    end
+
+    def add_all(items)
+      items.each do |i|
+        self << i
+      end
+    end
+
+    def flush
+      process_batch if @current_batch.present?
+    end
+
+    protected
+
+    def process_batch
+      @block.call(@current_batch)
+      @current_batch = []
+    end
+  end
+end

data/lib/canvas_sync/concerns/ability_helper.rb

@@ -0,0 +1,72 @@
+begin
+  require "panda_pal"
+  require "panda_pal/concerns/ability_helper"
+rescue LoadError
+end
+
+module CanvasSync::Concerns
+  module AbilityHelper
+    extend ActiveSupport::Concern
+
+    if defined?(PandaPal::Concerns::AbilityHelper)
+      include PandaPal::Concerns::AbilityHelper
+    else
+      def panda_pal_session
+        raise "This feature was moved to PandaPal as of CanvasSync 0.20.0/PandaPal 5.9.9. You should update to PandaPal >= 5.9.9."
+      end
+    end
+
+    # Middle Domain
+
+    def launch_context
+      @launch_context ||= begin
+        if panda_pal_session.lti_launch_placement == "global_navigation"
+          :global
+        elsif panda_pal_session.get_lti_cust_param('custom_canvas_course_id').present?
+          ::Course.find_by(canvas_id: panda_pal_session.get_lti_cust_param('custom_canvas_course_id'))
+        else
+          ::Account.find_by(canvas_id: panda_pal_session.get_lti_cust_param('custom_canvas_account_id'))
+        end
+      end
+    end
+
+    def launch_account
+      @launch_account ||= launch_context.respond_to?(:account) ?
+                            launch_context.account :
+                            ::Account.find_by(canvas_id: panda_pal_session.get_lti_cust_param('custom_canvas_account_id'))
+    end
+
+    # CanvasSync Domain
+
+    def canvas_permissions
+      panda_pal_session[:canvas_permissions] ||= ::Role.joined_permissions(canvas_roles)
+    end
+
+    def canvas_roles
+      @canvas_roles ||= Role.for_labels(panda_pal_session.canvas_role_labels, launch_account)
+    end
+
+    def canvas_root_account_roles
+      role_labels = panda_pal_session.canvas_account_role_labels('self')
+      ::Role.for_labels(role_labels, ::Account.find_by(canvas_parent_account_id: nil))
+    end
+
+    def canvas_account_roles
+      canvas_roles.where(base_role_type: 'AccountMembership')
+    end
+
+    def canvas_course_roles
+      canvas_roles.where.not(base_role_type: 'AccountMembership')
+    end
+
+    def canvas_super_user?
+      panda_pal_session.cache(:canvas_super_user?) do
+        panda_pal_session.canvas_site_admin? || (panda_pal_session.canvas_account_role_labels(:root) & ["AccountAdmin", "Account Admin"]).present?
+      end
+    end
+
+    def canvas_user_id
+      user&.canvas_id || panda_pal_session.get_lti_cust_param('canvas_user_id')
+    end
+  end
+end

data/lib/canvas_sync/concerns/account/ancestry.rb

@@ -11,6 +11,8 @@ module CanvasSync::Concerns
       extend ActiveSupport::Concern
       include CanvasSync::Record
 
+      CanvasSync::Record.define_feature self, default: ->{ column_names.include?("ancestry") rescue nil }
+
       included do
         has_ancestry
         before_save :relink_ancestry, if: :canvas_parent_account_id_changed?

data/lib/canvas_sync/concerns/account/base.rb

@@ -0,0 +1,15 @@
+module CanvasSync::Concerns
+  module Account
+    module Base
+      extend ActiveSupport::Concern
+
+      CanvasSync::Record.define_feature self, default: true
+
+      class_methods do
+        def root_account
+          where(canvas_parent_account_id: nil).last
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/concerns/api_syncable.rb

@@ -1,7 +1,7 @@
 module CanvasSync::Concerns
   module ApiSyncable
     extend ActiveSupport::Concern
-    NON_EXISTANT_ERRORS = [Faraday::Error::ResourceNotFound, Footrest::HttpError::NotFound]
+    NON_EXISTANT_ERRORS = [Faraday::ResourceNotFound, Footrest::HttpError::NotFound]
 
     class_methods do
       def find_or_fetch(canvas_id, save: false, retries: 1, **kwargs)
@@ -30,27 +30,27 @@ module CanvasSync::Concerns
       end
 
       def bulk_sync_from_api_result(api_array, conflict_target: :canvas_id, import_args: {}, all_pages: true, batch_size: 1000)
-        columns = api_sync_options.keys
+        columns = api_sync_options[:field_map].keys
 
         update_conditions = {
-          condition: Importers::BulkImporter.condition_sql(self, columns),
+          condition: CanvasSync::Importers::BulkImporter.condition_sql(self, columns),
           columns: columns,
         }
         update_conditions[:conflict_target] = conflict_target if conflict_target.present?
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
 
         if all_pages
-          batcher = BatchProcessor.new(of: batch_size) do |batch|
+          batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
             import(columns, batch, options)
           end
           api_array.all_pages_each do |api_item|
-            item = new.assign_from_api_params(api_items)
+            item = new.assign_from_api_params(api_item)
             batcher << item
           end
           batcher.flush
         else
           items = api_array.map do |api_item|
-            new.assign_from_api_params(api_items)
+            new.assign_from_api_params(api_item)
           end
           import(columns, batch, options)
         end
@@ -90,10 +90,12 @@ module CanvasSync::Concerns
       private
 
       def api_sync_race_create!(inst, save: true)
-        inst = find_or_initialize_by(canvas_id: inst) unless inst.is_a?(self)
-        yield inst
-        inst.save! if save && inst.changed?
-        inst
+        transaction(requires_new: true) do
+          inst = find_or_initialize_by(canvas_id: inst) unless inst.is_a?(self)
+          yield inst
+          inst.save! if save && inst.changed?
+          inst
+        end
       rescue ActiveRecord::RecordNotUnique
         inst = find_by(canvas_id: inst.canvas_id)
         yield inst
@@ -155,6 +157,11 @@ module CanvasSync::Concerns
       end
 
       apply_block = options[:process_response]
+
+      if self.class.column_names.include?("canvas_synced_at")
+        self.canvas_synced_at = mapped_params[:canvas_synced_at] = DateTime.now
+      end
+
       if apply_block.present?
         case apply_block.arity
         when 1

data/lib/canvas_sync/concerns/live_event_sync.rb

@@ -0,0 +1,46 @@
+
+module CanvasSync::Concerns
+  module LiveEventSync
+    extend ActiveSupport::Concern
+
+    included do
+      define_model_callbacks :process_live_event
+    end
+
+    class_methods do
+      def cs_internal_process_live_event(event)
+        meta = event[:metadata]
+        payload = event[:payload] || event[:body]
+
+        canvas_id = payload[:id] || payload[:"#{name.underscore}_id"]
+        inst = self.find_or_initialize_by(canvas_id: canvas_id)
+        model, _, subtype = meta[:event_name].rpartition('_')
+
+        result = inst.run_callbacks(:process_live_event) do
+          inst.process_live_event(subtype.to_sym, payload, meta)
+        end
+
+        inst.save! if result != false && inst.changed?
+      end
+    end
+
+    def process_live_event(event_type, payload, metadata)
+      api_response = request_from_api
+      assign_from_api_params(api_response)
+    end
+  end
+
+  CanvasSync::LiveEvents.listen do |event|
+    meta = event[:metadata]
+    payload = event[:body]
+
+    event_type = meta[:event_name]
+
+    model, _, subtype = event_type.rpartition('_')
+    mcls = model.classify.constantize rescue nil
+
+    if mcls.present? && mcls < LiveEventSync
+      mcls.cs_internal_process_live_event(event)
+    end
+  end
+end

data/lib/canvas_sync/concerns/role/base.rb

@@ -0,0 +1,57 @@
+module CanvasSync::Concerns
+  module Role
+    module Base
+      extend ActiveSupport::Concern
+
+      DEFAULT_ROLE_LABELS = %w[TeacherEnrollment TaEnrollment StudentEnrollment DesignerEnrollment ObserverEnrollment].freeze
+
+      CanvasSync::Record.define_feature self, default: true
+
+      class_methods do
+        def for_labels(labels, account)
+          built_ins = []
+          labels = labels.split(',') if labels.is_a?(String)
+          custom_labels = Array(labels).reject do |l|
+            if DEFAULT_ROLE_LABELS.include?(l)
+              built_ins << l
+            elsif l == 'Account Admin' || l == 'AccountAdmin'
+              built_ins << 'AccountMembership'
+            else
+              next
+            end
+            true
+          end
+
+          account_ids = Rails.cache.fetch([self.class.name, "AccountAncestry", account.canvas_id], expires_in: 6.hours) do
+            if account.respond_to?(:path_ids)
+              account.path.pluck(:canvas_id)
+            else
+              [].tap do |pids|
+                acc = account
+                loop do
+                  break unless acc
+                  pids.unshift(acc.canvas_id)
+                  acc = acc.canvas_parent
+                end
+              end
+            end
+          end
+
+          where(workflow_state: 'built_in', base_role_type: built_ins)
+            .or(where.not(workflow_state: 'built_in').where(label: custom_labels, canvas_account_id: account_ids))
+        end
+
+        def joined_permissions(roles)
+          final = {}
+          roles.each do |role|
+            role.permissions.each do |perm_name, perm|
+              final[perm_name] = false if final[perm_name].nil?
+              final[perm_name] = true if perm['enabled'] == true
+            end
+          end
+          final
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/concerns/sync_mapping.rb

@@ -0,0 +1,120 @@
+module CanvasSync::Concerns
+  module SyncMapping
+    extend ActiveSupport::Concern
+
+    def self.mapping_for(model, key = nil)
+      model.try(:get_sync_mapping, key) || Mapping.default_mappings[key || Mapping.normalize_model_name(model)]
+    end
+
+    class_methods do
+      def sync_mapping(key = nil, reset: false, &blk)
+        key ||= Mapping.normalize_model_name(self)
+        key = key.to_s
+        existing_map = get_sync_mapping(key)
+        mapper = Mapping.new(existing_map&.deep_dup || {}.with_indifferent_access)
+        mapper.reset_links if reset
+        mapper.instance_exec(&blk)
+        @sync_mappings[key] = mapper.map_def.freeze
+      end
+
+      def get_sync_mapping(key = nil)
+        key ||= Mapping.normalize_model_name(self)
+        key = key.to_s
+        @sync_mappings ||= {}
+        @sync_mappings[key] || superclass.try(:get_sync_mapping, key) || Mapping.default_for(key)
+      end
+    end
+
+    class Mapping
+      attr_reader :map_def
+
+      def initialize(map_def, model: nil)
+        @model = model
+        @map_def = map_def
+        @map_def[:conflict_target] ||= []
+        @map_def[:columns] ||= {}
+      end
+
+      def self.normalize_model_name(model)
+        model = model.name unless model.is_a?(String)
+        model.pluralize.underscore
+      end
+
+      def self.default_for(key)
+        default_mappings[key]
+      end
+
+      def self.default_mappings
+        @mappings ||= begin
+          maps = {}
+          default_v1_mappings.each do |mname, legacy|
+            m = maps[mname] = {}
+
+            m[:conflict_target] = Array(legacy[:conflict_target]).map(&:to_sym).map do |lct|
+              legacy[:report_columns][lct][:database_column_name]
+            end
+
+            m[:columns] = {}
+            legacy[:report_columns].each do |rcol, opts|
+              m[:columns][opts[:database_column_name]] = opts.except(:database_column_name).merge!(
+                report_column: rcol,
+              ).freeze
+            end
+          end
+          maps.with_indifferent_access.freeze
+        end
+      end
+
+      def self.default_v1_mappings
+        @legacy_mappings ||= begin
+          mapping = YAML.load_file(File.join(__dir__, '../processors', "model_mappings.yml")).deep_symbolize_keys!
+          override_filepath = Rails.root.join("config/canvas_sync_provisioning_mapping.yml")
+
+          if File.file?(override_filepath)
+            override = YAML.load_file(override_filepath).deep_symbolize_keys!
+            mapping = mapping.merge(override)
+          end
+
+          mapping.freeze
+        end
+      end
+
+      def conflict_target(*columns)
+        if columns.count == 0
+          @map_def[:conflict_target]
+        else
+          @map_def[:conflict_target] = columns.flatten.compact
+        end
+      end
+
+      def reset_links
+        @map_def[:columns] = {}.with_indifferent_access
+      end
+
+      def unlink_column(key)
+        @map_def[:columns].delete(key)
+      end
+
+      def link_column(m = {}, type: nil, **kwargs, &blk)
+        if m.is_a?(Hash)
+          m = m.merge(kwargs)
+          raise "Hash should have exactly 1 entry" if m && m.count != 1
+          @map_def[:columns][m.values[0]] = {
+            report_column: m.keys[0],
+            type: type,
+            transform: blk,
+          }
+        elsif m.is_a?(Symbol)
+          raise "Unrecognized keyword arguments" if kwargs.present?
+          @map_def[:columns][m] = {
+            report_column: m,
+            type: type,
+            transform: blk,
+          }
+        else
+          raise "Cannot handle argument of type #{m.class}"
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/engine.rb

@@ -1,9 +1,25 @@
 require "rails"
+require 'chronic_duration'
 
 module CanvasSync
   class Engine < ::Rails::Engine
     isolate_namespace CanvasSync
 
+    initializer "canvas_sync.safe_yaml_classes" do |app|
+      app.config.active_record.yaml_column_permitted_classes ||= []
+      app.config.active_record.yaml_column_permitted_classes |= [Symbol, ActiveSupport::HashWithIndifferentAccess]
+
+      Rails.application.config.after_initialize do
+        if ActiveRecord::Base.respond_to?(:yaml_column_permitted_classes)
+          ActiveRecord::Base.yaml_column_permitted_classes |= app.config.active_record.yaml_column_permitted_classes
+        end
+        if ActiveRecord.respond_to?(:yaml_column_permitted_classes)
+          ActiveRecord.yaml_column_permitted_classes |= app.config.active_record.yaml_column_permitted_classes
+        end
+      end
+    rescue
+    end
+
     initializer :append_migrations do |app|
       config.paths["db/migrate"].expanded.each do |expanded_path|
         app.config.paths["db/migrate"] << expanded_path
@@ -11,5 +27,69 @@ module CanvasSync
       # Apartment will modify this, but it doesn't fully support engine migrations, so we'll reset it here
       ActiveRecord::Migrator.migrations_paths = Rails.application.paths["db/migrate"].to_a
     end
+
+    RETENTION_TYPE = {
+      type: 'string',
+      validate: ->(value, *args, errors:, **kwargs) {
+        origExc = ChronicDuration.raise_exceptions
+        ChronicDuration.raise_exceptions = true
+        begin
+          ChronicDuration.parse(value) unless value == nil
+          nil
+        rescue ChronicDuration.DurationParseError
+          errors << "<path> must be nil or a parseable duration"
+        ensure
+          ChronicDuration.raise_exceptions = origExc
+        end
+      }
+    }
+
+    initializer 'canvas_sync.global_methods' do
+      next if defined?(canvas_sync_client)
+
+      require 'panda_pal'
+
+      class ::Object
+        def canvas_sync_client(account_id = nil)
+          org = PandaPal::Organization.current
+          org = org.platform_api(PandaPal::Platform::Canvas) if (PandaPal::Organization.instance_method(:platform_api) rescue nil)
+          Bearcat::Client.new(
+            prefix: org.canvas_url,
+            token: org.canvas_api_token,
+          )
+        end
+      end
+    rescue LoadError
+    end
+
+    initializer :integrate_pandapal do
+      require 'panda_pal'
+
+      Rails.application.reloader.to_prepare do
+        if PandaPal::Organization.respond_to?(:scheduled_task)
+          if PandaPal::Organization.respond_to?(:define_setting)
+            PandaPal::Organization.define_setting(:canvas_sync, {
+              type: 'Hash',
+              required: false,
+              properties: {
+                job_log_retention: { **RETENTION_TYPE },
+                sync_batch_retention: { **RETENTION_TYPE },
+              }
+            })
+          end
+
+          unless PandaPal::Organization.task_scheduled?(:clean_canvas_sync_logs)
+            PandaPal::Organization.scheduled_task '0 0 3 * * *', :clean_canvas_sync_logs do
+              job_log_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :job_log_retention) || '3 months', keep_zero: true).seconds.ago
+              JobLog.where('updated_at < ?', job_log_retention).delete_all
+
+              sync_batch_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :sync_batch_retention) || '6 months', keep_zero: true).seconds.ago
+              SyncBatch.where('updated_at < ?', sync_batch_retention).delete_all
+            end
+          end
+        end
+      end
+    rescue LoadError
+    end
   end
 end

data/lib/canvas_sync/generators/install_generator.rb

@@ -48,6 +48,7 @@ module CanvasSync
       models.each do |model|
         migration_template "migrations/create_#{model}.rb", "db/migrate/create_#{model}.rb"
         template "models/#{model.singularize}.rb", "app/models/#{model.singularize}.rb"
+      rescue
       end
     end
   end