canvas_sync 0.17.18 → 0.17.23.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30e1a16308bbc8795dc671c19f35d29577665213261829d3180a9efeefab4adf
-  data.tar.gz: bade0d8907b22bf252b85ba4067798ebd3b6582deed06adfccf7a8951a6164b4
+  metadata.gz: 0d5da2edfe978cc9fec69714ce7a1bd90cce3977023478473a7999143e78df90
+  data.tar.gz: 510f341eb1d090275b302e971a6f26c59e046da7699cfaee567a34c23ce751c8
 SHA512:
-  metadata.gz: 4a155b415154694a42c218bd5ef91f21ca7cdcc81967cc2f2c16a180eebfb32d1b279c72ea2eab58beaabb8e8a8905fe4ef4086ce758c9ee14b1583eccf06849
-  data.tar.gz: 3269c39188c086e7b3b03d6280414fbdc7a32c6780af5744da08f3f9432f4eb5ed993c3af4d22e2fe477d92ca8e4d705c62ba9e9005559a9e1145922f565aa4f
+  metadata.gz: 891a38e3bd240de7c5c248d14bc21f84bc1336a19fb1509e74769076d7059bc4e3a2b7b8764cafe3aba8b6127d2278241a30b8f78f0735467408c269710dab31
+  data.tar.gz: fc71f424836699e98cfa39ee435e919e81cb9bcc12deef30f4597806746ec8047136efc180db31ad906dc309d62526584fbdd17db04d2fe7b9b7495402f869a9

data/README.md

@@ -190,7 +190,30 @@ Overrides are useful for two scenarios:
 - You have an existing application where the column names do not match up with what CanvasSync expects
 - You want to sync some other column in the report that CanvasSync is not configured to sync
 
-In order to create an override, place a file called `canvas_sync_provisioning_mapping.yml` in your Rails `config` directory. Define the tables and columns you want to override using the following format:
+Mappings can be modified by editing the Model class like such:
+```ruby
+class User < ApplicationRecord
+  include CanvasSync::Record
+
+  sync_mapping(reset: false) do # `reset: false` is the default
+    # The mapping can be totally cleared with `reset: true` in the `sync_mapping` call, or like such:
+    reset_links
+
+    # Add a new column:
+    link_column :column_in_report => :column_in_database, type: :datetime
+
+    # If the column name on the report and in the DB are the same, a shorthand can be used:
+    link_column :omit_from_final_grade, type: :datetime
+
+    # If the defaults define a column you don't want synced, you can remove it from the mapping:
+    unlink_column :column_in_database
+  end
+
+  # ...
+end
+```
+
+You can also create a file called `canvas_sync_provisioning_mapping.yml` in your Rails `config` directory. However, this approach requires you to re-specify the complete table in order to modify a table. Define the tables and columns you want to override using the following format:
 
 ```ruby
 users:
@@ -383,6 +406,27 @@ Available config options (if you add more, please update this!):
 
 * `config.classes_to_only_log_errors_on` - use this if you are utilizing the `CanvasSync::JobLog` table, but want certain classes to only persist in the `job_logs` table if an error is encountered. This is useful if you've got a very frequently used job that's filling up your database, and only really care about tracking failures.
 
+## Global Options
+You can pass in global_options to a job chain.  Global options are added to the batch_context and referenced by
+various internal processes.
+
+Pass global options into a job chain, using the options param nested in a :global key.
+options: { global: {...} }
+
+report_timeout (integer):             Number of days until a Canvas report should timeout.  Default is 1.
+report_compilation_timeout (integer): Number of days until a Canvas report should timeout.  Default is 1 hour.
+                                      You can likely pass a float to achieve sub-day timeouts, but not tested.
+report_max_tries (integer):           The number of times to attempt a report before giving up.  A report is considered failed
+                                      if it has an 'error' status in Canvas or is deleted.
+
+This is an example job chain with global options:
+job_chain = CanvasSync.default_provisioning_report_chain(
+  MODELS_TO_SYNC,
+  term_scope: :active,
+  full_sync_every: 'sunday',
+  options: { global: { report_timeout: 2 } }
+)
+
 ## Handling Job errors
 
 If you need custom handling for when a CanvasSync Job fails, you can add an `:on_failure` option to you Job Chain's `:global_options`.

data/lib/canvas_sync/concerns/sync_mapping.rb

@@ -0,0 +1,112 @@
+module CanvasSync::Concerns
+  module SyncMapping
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def sync_mapping(key = nil, reset: false, &blk)
+        key ||= Mapping.normalize_model_name(self)
+        key = key.to_s
+        existing_map = get_sync_mapping(key)
+        mapper = Mapping.new(existing_map&.deep_dup || {}.with_indifferent_access)
+        mapper.reset_links if reset
+        mapper.instance_exec(&blk)
+        @sync_mappings[key] = mapper.map_def.freeze
+      end
+
+      def get_sync_mapping(key = nil)
+        key ||= Mapping.normalize_model_name(self)
+        key = key.to_s
+        @sync_mappings ||= {}
+        @sync_mappings[key] || superclass.try(:get_sync_mapping, key) || Mapping.default_for(key)
+      end
+    end
+
+    class Mapping
+      attr_reader :map_def
+
+      def initialize(map_def = {}, model: nil)
+        @map_def = map_def
+        @model = model
+      end
+
+      def self.normalize_model_name(model)
+        model = model.name unless model.is_a?(String)
+        model.pluralize.underscore
+      end
+
+      def self.default_for(key)
+        default_mappings[key]
+      end
+
+      def self.default_mappings
+        @mappings ||= begin
+          maps = {}
+          default_v1_mappings.each do |mname, legacy|
+            m = maps[mname] = {}
+
+            m[:conflict_target] = Array(legacy[:conflict_target]).map(&:to_sym).map do |lct|
+              legacy[:report_columns][lct][:database_column_name]
+            end
+
+            m[:report_columns] = {}
+            legacy[:report_columns].each do |rcol, opts|
+              m[:report_columns][opts[:database_column_name]] = opts.except(:database_column_name).merge!(
+                report_column: rcol,
+              ).freeze
+            end
+          end
+          maps.with_indifferent_access.freeze
+        end
+      end
+
+      def self.default_v1_mappings
+        @legacy_mappings ||= begin
+          mapping = YAML.load_file(File.join(__dir__, '../processors', "model_mappings.yml")).deep_symbolize_keys!
+          override_filepath = Rails.root.join("config/canvas_sync_provisioning_mapping.yml")
+
+          if File.file?(override_filepath)
+            override = YAML.load_file(override_filepath).deep_symbolize_keys!
+            mapping = mapping.merge(override)
+          end
+
+          mapping.freeze
+        end
+      end
+
+      def conflict_target(*columns)
+        if columns.count == 0
+          @map_def[:conflict_target]
+        else
+          @map_def[:conflict_target] = columns.flatten.compact
+        end
+      end
+
+      def reset_links
+        @map_def = {}
+      end
+
+      def unlink_column(key)
+        @map_def.delete(key)
+      end
+
+      def link_column(m, type: nil, &blk)
+        if m.is_a?(Hash)
+          raise "Hash should have exactly 1 entry" if m && m.count != 1
+          @map_def[:report_columns][m.values[0]] = {
+            report_column: m.keys[0],
+            type: type,
+            transform: blk,
+          }
+        elsif m.is_a?(Symbol)
+          @map_def[:report_columns][m] = {
+            report_column: m,
+            type: type,
+            transform: blk,
+          }
+        else
+          raise "Cannot handle argument of type #{m.class}"
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/generators/install_generator.rb

@@ -48,6 +48,7 @@ module CanvasSync
       models.each do |model|
         migration_template "migrations/create_#{model}.rb", "db/migrate/create_#{model}.rb"
         template "models/#{model.singularize}.rb", "app/models/#{model.singularize}.rb"
+      rescue
       end
     end
   end

data/lib/canvas_sync/generators/templates/migrations/create_grading_period_groups.rb

@@ -0,0 +1,22 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
+  def change
+    create_table :grading_period_groups do |t|
+      t.bigint :canvas_id, null: false
+      t.bigint :canvas_course_id
+      t.bigint :canvas_account_id
+      t.string :title
+      t.boolean :weighted
+      t.boolean :display_totals_for_all_grading_periods
+
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :grading_period_groups, :canvas_id, unique: true
+    add_index :grading_period_groups, :canvas_course_id
+    add_index :grading_period_groups, :canvas_account_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_grading_periods.rb

@@ -0,0 +1,22 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateGradingPeriods < ActiveRecord::Migration[5.1]
+  def change
+    create_table :grading_periods do |t|
+      t.bigint :canvas_id, null: false
+      t.string :title
+      t.float :weight
+      t.datetime :start_date
+      t.datetime :end_date
+      t.datetime :close_date
+      t.bigint :canvas_grading_period_group_id
+
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :grading_periods, :canvas_id, unique: true
+    add_index :grading_periods, :canvas_grading_period_group_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_user_observers.rb

@@ -0,0 +1,17 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateUserObservers < ActiveRecord::Migration[5.1]
+  def change
+    create_table :user_observers do |t|
+      t.bigint :observing_user_id
+      t.bigint :observed_user_id
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :user_observers, [:observed_user_id, :observing_user_id], unique: true
+    add_index :user_observers, :observing_user_id
+    add_index :user_observers, :observed_user_id
+  end
+end

data/lib/canvas_sync/generators/templates/models/grading_period.rb

@@ -0,0 +1,8 @@
+# <%= autogenerated_model_warning %>
+
+class GradingPeriod < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :canvas_grading_period_group_id, optional: true
+end

data/lib/canvas_sync/generators/templates/models/grading_period_group.rb

@@ -0,0 +1,9 @@
+# <%= autogenerated_model_warning %>
+
+class GradingPeriodGroup < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
+  belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
+end

data/lib/canvas_sync/generators/templates/models/user_observer.rb

@@ -0,0 +1,11 @@
+# <%= autogenerated_model_warning %>
+
+class UserObserver < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+
+  belongs_to :observing_user, primary_key: :canvas_id, foreign_key: :observing_user_id, class_name: 'User', optional: true
+  belongs_to :observed_user, primary_key: :canvas_id, foreign_key: :observed_user_id, class_name: 'User', optional: true
+end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -24,30 +24,41 @@ module CanvasSync
       end
 
       def self.perform_in_batches(report_file_path, mapping, klass, conflict_target, import_args: {})
-        csv_column_names = mapping.keys
-        database_column_names = mapping.values.map { |value| value[:database_column_name] }
-        database_conflict_column_name = conflict_target ? mapping[conflict_target][:database_column_name] : nil
+        csv_column_names = mapping.values.map { |value| value[:report_column].to_s }
+        database_column_names = mapping.keys
+
+        conflict_target = Array(conflict_target).map(&:to_s)
+        conflict_target_indices = conflict_target.map{|ct| database_column_names.index(ct) }
 
         row_ids = {}
         batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
           row_ids = {}
-          perform_import(klass, database_column_names, batch, database_conflict_column_name, import_args)
+          perform_import(klass, database_column_names, batch, conflict_target, import_args)
         end
 
         row_buffer_out = ->(row) {
-          if conflict_target
-            next if row_ids[row[conflict_target]]
-            row_ids[row[conflict_target]] = true
+          formatted_row = mapping.map do |db_col, col_def|
+            value = nil
+            value = row[col_def[:report_column]] if col_def[:report_column]
+
+            if col_def[:type]
+              if col_def[:type].to_sym == :datetime
+                # TODO: add some timezone config to the mapping.
+                # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
+                value = DateTime.parse(value).utc rescue nil # rubocop:disable Style/RescueModifier
+              end
+            end
+
+            value = col_def[:transform].call(value, row) if col_def[:transform]
+
+            value
           end
 
-          formatted_row = csv_column_names.map do |column|
-            if mapping[column][:type].to_sym == :datetime
-              # TODO: add some timezone config to the mapping.
-              # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
-              DateTime.parse(row[column]).utc rescue nil # rubocop:disable Style/RescueModifier
-            else
-              row[column]
-            end
+          if conflict_target.present?
+            key = conflict_target_indices.map{|ct| formatted_row[ct] }
+            next if row_ids[key]
+
+            row_ids[key] = true
           end
 
           batcher << formatted_row
@@ -79,7 +90,7 @@ module CanvasSync
           condition: condition_sql(klass, columns, import_args[:sync_start_time]),
           columns: columns
         }
-        update_conditions[:conflict_target] = conflict_target if conflict_target
+        update_conditions[:conflict_target] = conflict_target if conflict_target.present?
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -40,7 +40,7 @@ module CanvasSync
       def insert_at(position, new_jobs)
         chain = self.class.get_chain_parameter(base_job)
         new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
-        chain.insert(-1, *new_jobs)
+        chain.insert(position, *new_jobs)
       end
 
       def insert(new_jobs, **kwargs)

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -46,7 +46,7 @@ module CanvasSync
           m = Regexp.last_match
           day = m[1]
           skip = m[2] || "1"
-          Date.new.send(:"#{day}?") && last_full_sync.end_of_day <= (skip.to_i.weeks.ago.end_of_day)
+          DateTime.now.send(:"#{day}?") && last_full_sync.end_of_day <= (skip.to_i.weeks.ago.end_of_day)
         when opt.match?(%r{^(\d+)\%$})
           m = Regexp.last_match
           rand(100) < m[1].to_i

data/lib/canvas_sync/jobs/report_checker.rb

@@ -4,8 +4,9 @@ module CanvasSync
     # Re-enqueues itself if the report is still processing on Canvas.
     # Enqueues the ReportProcessor when the report has completed.
     class ReportChecker < CanvasSync::Job
-      REPORT_TIMEOUT = 12.hours
+      REPORT_TIMEOUT = 24.hours
       COMPILATION_TIMEOUT = 1.hour
+      MAX_TRIES = 3
 
       # @param report_name [Hash] e.g., 'provisioning_csv'
       # @param report_id [Integer]
@@ -13,6 +14,7 @@ module CanvasSync
       # @param options [Hash] hash of options that will be passed to the job processor
       # @return [nil]
       def perform(report_name, report_id, processor, options, checker_context = {}) # rubocop:disable Metrics/AbcSize
+        max_tries = options[:report_max_tries] || batch_context[:report_max_tries] || MAX_TRIES
         account_id = options[:account_id] || batch_context[:account_id] || "self"
         report_status = CanvasSync.get_canvas_sync_client(batch_context)
                                   .report_status(account_id, report_name, report_id)
@@ -27,9 +29,17 @@ module CanvasSync
             report_id,
           )
         when "error", "deleted"
-          message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}" # rubocop:disable Metrics/LineLength
+          checker_context[:failed_attempts] ||= 0
+          checker_context[:failed_attempts] += 1
+          failed_attempts = checker_context[:failed_attempts]
+          message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}, #{current_organization.name}.  This report has now failed #{checker_context[:failed_attempts]} time." # rubocop:disable Metrics/LineLength
           Rails.logger.error(message)
-          raise message
+          if failed_attempts >= max_tries
+            Rails.logger.error("This report has failed #{failed_attempts} times.  Giving up.")
+            raise message
+          else
+            restart_report(options, report_name, processor, checker_context)
+          end
         else
           report_timeout = parse_timeout(options[:report_timeout] || batch_context[:report_timeout] || REPORT_TIMEOUT)
           if timeout_met?(options[:sync_start_time], report_timeout)
@@ -51,7 +61,7 @@ module CanvasSync
               report_id,
               processor,
               options,
-              checker_context,
+              checker_context
             )
         end
       end
@@ -66,6 +76,29 @@ module CanvasSync
       def parse_timeout(val)
         val
       end
+
+      def restart_report(options, report_name, processor, checker_context)
+        account_id = options[:account_id] || batch_context[:account_id] || "self"
+        options[:sync_start_time] = DateTime.now.utc.iso8601
+        new_context = {}
+        new_context[:failed_attempts] = checker_context[:failed_attempts]
+        report_id = start_report(account_id, report_name, options[:report_params])
+        CanvasSync::Jobs::ReportChecker
+            .set(wait: report_checker_wait_time)
+            .perform_later(
+              report_name,
+              report_id,
+              processor,
+              options,
+              new_context
+            )
+      end
+
+      def start_report(account_id, report_name, report_params)
+        report = CanvasSync.get_canvas_sync_client(batch_context)
+                           .start_report(account_id, report_name, report_params)
+        report["id"]
+      end
     end
   end
 end