canvas_sync 0.17.19 → 0.17.23.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d995ce469310b7b54158745305c968edd4c7a910bcad87891ffabfa29cd22a30
-  data.tar.gz: 83a0bb75ceb290402137f2531e1925d4872a0e272fcabf75a6c3451b3de5f721
+  metadata.gz: c53a6ea9523ce389c70e26035183573de72ee06c547e380842216b879a6d8bdc
+  data.tar.gz: 166c315f6ba7b0efa337c55a56b8cd418e080f8f98393c4eaed1fd9fdc75a862
 SHA512:
-  metadata.gz: 9287196092064977d34259de638cfd41cf189f6dc9b6a97c6fd507d2555f8f3b0527643d38991765d56bf428856407608ef0c1834b48b6a6dd3201ed232b9879
-  data.tar.gz: a751ad1e08d3feb2716e41d4c552f983f2d83590321dc4d1cc2a18d2f800859587fa1884272bbdbaff0cc4aefe867c0af8f0dfb24c2dc6aba25190e7a190449a
+  metadata.gz: f7137c46728c087f7e53902099744c4545db67d01da9117f243bbdf9e6ab099c5668a9f9be44752b6d4a4ca4208d1e98ebb2831e777be531416f01cc3befd21c
+  data.tar.gz: 62eecd4a5581297f95ee82c2e81c5c7821565defc9885024153ab17a708c00b5eda22f18e83773decb1884a73d6a7a74453d108c26e50f8f868934cb03a60e54

data/README.md

@@ -190,7 +190,30 @@ Overrides are useful for two scenarios:
 - You have an existing application where the column names do not match up with what CanvasSync expects
 - You want to sync some other column in the report that CanvasSync is not configured to sync
 
-In order to create an override, place a file called `canvas_sync_provisioning_mapping.yml` in your Rails `config` directory. Define the tables and columns you want to override using the following format:
+Mappings can be modified by editing the Model class like such:
+```ruby
+class User < ApplicationRecord
+  include CanvasSync::Record
+
+  sync_mapping(reset: false) do # `reset: false` is the default
+    # The mapping can be totally cleared with `reset: true` in the `sync_mapping` call, or like such:
+    reset_links
+
+    # Add a new column:
+    link_column :column_in_report => :column_in_database, type: :datetime
+
+    # If the column name on the report and in the DB are the same, a shorthand can be used:
+    link_column :omit_from_final_grade, type: :datetime
+
+    # If the defaults define a column you don't want synced, you can remove it from the mapping:
+    unlink_column :column_in_database
+  end
+
+  # ...
+end
+```
+
+You can also create a file called `canvas_sync_provisioning_mapping.yml` in your Rails `config` directory. However, this approach requires you to re-specify the complete table in order to modify a table. Define the tables and columns you want to override using the following format:
 
 ```ruby
 users:
@@ -383,6 +406,27 @@ Available config options (if you add more, please update this!):
 
 * `config.classes_to_only_log_errors_on` - use this if you are utilizing the `CanvasSync::JobLog` table, but want certain classes to only persist in the `job_logs` table if an error is encountered. This is useful if you've got a very frequently used job that's filling up your database, and only really care about tracking failures.
 
+## Global Options
+You can pass in global_options to a job chain.  Global options are added to the batch_context and referenced by
+various internal processes.
+
+Pass global options into a job chain, using the options param nested in a :global key.
+options: { global: {...} }
+
+report_timeout (integer):             Number of days until a Canvas report should timeout.  Default is 1.
+report_compilation_timeout (integer): Number of days until a Canvas report should timeout.  Default is 1 hour.
+                                      You can likely pass a float to achieve sub-day timeouts, but not tested.
+report_max_tries (integer):           The number of times to attempt a report before giving up.  A report is considered failed
+                                      if it has an 'error' status in Canvas or is deleted.
+
+This is an example job chain with global options:
+job_chain = CanvasSync.default_provisioning_report_chain(
+  MODELS_TO_SYNC,
+  term_scope: :active,
+  full_sync_every: 'sunday',
+  options: { global: { report_timeout: 2 } }
+)
+
 ## Handling Job errors
 
 If you need custom handling for when a CanvasSync Job fails, you can add an `:on_failure` option to you Job Chain's `:global_options`.

data/lib/canvas_sync/concerns/sync_mapping.rb

@@ -0,0 +1,114 @@
+module CanvasSync::Concerns
+  module SyncMapping
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def sync_mapping(key = nil, reset: false, &blk)
+        key ||= Mapping.normalize_model_name(self)
+        key = key.to_s
+        existing_map = get_sync_mapping(key)
+        mapper = Mapping.new(existing_map&.deep_dup || {}.with_indifferent_access)
+        mapper.reset_links if reset
+        mapper.instance_exec(&blk)
+        @sync_mappings[key] = mapper.map_def.freeze
+      end
+
+      def get_sync_mapping(key = nil)
+        key ||= Mapping.normalize_model_name(self)
+        key = key.to_s
+        @sync_mappings ||= {}
+        @sync_mappings[key] || superclass.try(:get_sync_mapping, key) || Mapping.default_for(key)
+      end
+    end
+
+    class Mapping
+      attr_reader :map_def
+
+      def initialize(map_def, model: nil)
+        @model = model
+        @map_def = map_def
+        @map_def[:conflict_target] ||= []
+        @map_def[:report_columns] ||= {}
+      end
+
+      def self.normalize_model_name(model)
+        model = model.name unless model.is_a?(String)
+        model.pluralize.underscore
+      end
+
+      def self.default_for(key)
+        default_mappings[key]
+      end
+
+      def self.default_mappings
+        @mappings ||= begin
+          maps = {}
+          default_v1_mappings.each do |mname, legacy|
+            m = maps[mname] = {}
+
+            m[:conflict_target] = Array(legacy[:conflict_target]).map(&:to_sym).map do |lct|
+              legacy[:report_columns][lct][:database_column_name]
+            end
+
+            m[:report_columns] = {}
+            legacy[:report_columns].each do |rcol, opts|
+              m[:report_columns][opts[:database_column_name]] = opts.except(:database_column_name).merge!(
+                report_column: rcol,
+              ).freeze
+            end
+          end
+          maps.with_indifferent_access.freeze
+        end
+      end
+
+      def self.default_v1_mappings
+        @legacy_mappings ||= begin
+          mapping = YAML.load_file(File.join(__dir__, '../processors', "model_mappings.yml")).deep_symbolize_keys!
+          override_filepath = Rails.root.join("config/canvas_sync_provisioning_mapping.yml")
+
+          if File.file?(override_filepath)
+            override = YAML.load_file(override_filepath).deep_symbolize_keys!
+            mapping = mapping.merge(override)
+          end
+
+          mapping.freeze
+        end
+      end
+
+      def conflict_target(*columns)
+        if columns.count == 0
+          @map_def[:conflict_target]
+        else
+          @map_def[:conflict_target] = columns.flatten.compact
+        end
+      end
+
+      def reset_links
+        @map_def[:report_columns] = {}.with_indifferent_access
+      end
+
+      def unlink_column(key)
+        @map_def.delete(key)
+      end
+
+      def link_column(m, type: nil, &blk)
+        if m.is_a?(Hash)
+          raise "Hash should have exactly 1 entry" if m && m.count != 1
+          @map_def[:report_columns][m.values[0]] = {
+            report_column: m.keys[0],
+            type: type,
+            transform: blk,
+          }
+        elsif m.is_a?(Symbol)
+          @map_def[:report_columns][m] = {
+            report_column: m,
+            type: type,
+            transform: blk,
+          }
+        else
+          raise "Cannot handle argument of type #{m.class}"
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/generators/install_generator.rb

@@ -48,6 +48,7 @@ module CanvasSync
       models.each do |model|
         migration_template "migrations/create_#{model}.rb", "db/migrate/create_#{model}.rb"
         template "models/#{model.singularize}.rb", "app/models/#{model.singularize}.rb"
+      rescue
       end
     end
   end

data/lib/canvas_sync/generators/templates/migrations/create_grading_period_groups.rb

@@ -0,0 +1,22 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
+  def change
+    create_table :grading_period_groups do |t|
+      t.bigint :canvas_id, null: false
+      t.bigint :canvas_course_id
+      t.bigint :canvas_account_id
+      t.string :title
+      t.boolean :weighted
+      t.boolean :display_totals_for_all_grading_periods
+
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :grading_period_groups, :canvas_id, unique: true
+    add_index :grading_period_groups, :canvas_course_id
+    add_index :grading_period_groups, :canvas_account_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_grading_periods.rb

@@ -0,0 +1,22 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateGradingPeriods < ActiveRecord::Migration[5.1]
+  def change
+    create_table :grading_periods do |t|
+      t.bigint :canvas_id, null: false
+      t.string :title
+      t.float :weight
+      t.datetime :start_date
+      t.datetime :end_date
+      t.datetime :close_date
+      t.bigint :canvas_grading_period_group_id
+
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :grading_periods, :canvas_id, unique: true
+    add_index :grading_periods, :canvas_grading_period_group_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_user_observers.rb

@@ -0,0 +1,17 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateUserObservers < ActiveRecord::Migration[5.1]
+  def change
+    create_table :user_observers do |t|
+      t.bigint :observing_user_id
+      t.bigint :observed_user_id
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :user_observers, [:observed_user_id, :observing_user_id], unique: true
+    add_index :user_observers, :observing_user_id
+    add_index :user_observers, :observed_user_id
+  end
+end

data/lib/canvas_sync/generators/templates/models/grading_period.rb

@@ -0,0 +1,8 @@
+# <%= autogenerated_model_warning %>
+
+class GradingPeriod < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :canvas_grading_period_group_id, optional: true
+end

data/lib/canvas_sync/generators/templates/models/grading_period_group.rb

@@ -0,0 +1,9 @@
+# <%= autogenerated_model_warning %>
+
+class GradingPeriodGroup < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
+  belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
+end

data/lib/canvas_sync/generators/templates/models/user_observer.rb

@@ -0,0 +1,11 @@
+# <%= autogenerated_model_warning %>
+
+class UserObserver < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+
+  belongs_to :observing_user, primary_key: :canvas_id, foreign_key: :observing_user_id, class_name: 'User', optional: true
+  belongs_to :observed_user, primary_key: :canvas_id, foreign_key: :observed_user_id, class_name: 'User', optional: true
+end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -24,38 +24,43 @@ module CanvasSync
       end
 
       def self.perform_in_batches(report_file_path, mapping, klass, conflict_target, import_args: {})
-        csv_column_names = mapping.keys
-        database_column_names = mapping.values.map { |value| value[:database_column_name] }
+        csv_column_names = mapping.values.map { |value| value[:report_column].to_s }
+        database_column_names = mapping.keys
 
-        puts mapping.inspect
-
-        conflict_target = Array(conflict_target).map(&:to_sym)
-        database_conflict_column_name = conflict_target.map{|ct| mapping[ct][:database_column_name] }
+        conflict_target = Array(conflict_target).map(&:to_s)
+        conflict_target_indices = conflict_target.map{|ct| database_column_names.index(ct) }
 
         row_ids = {}
         batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
           row_ids = {}
-          perform_import(klass, database_column_names, batch, database_conflict_column_name, import_args)
+          perform_import(klass, database_column_names, batch, conflict_target, import_args)
         end
 
         row_buffer_out = ->(row) {
+          formatted_row = mapping.map do |db_col, col_def|
+            value = nil
+            value = row[col_def[:report_column]] if col_def[:report_column]
+
+            if col_def[:type]
+              if col_def[:type].to_sym == :datetime
+                # TODO: add some timezone config to the mapping.
+                # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
+                value = DateTime.parse(value).utc rescue nil # rubocop:disable Style/RescueModifier
+              end
+            end
+
+            value = col_def[:transform].call(value, row) if col_def[:transform]
+
+            value
+          end
+
           if conflict_target.present?
-            key = conflict_target.map{|ct| row[ct] }
+            key = conflict_target_indices.map{|ct| formatted_row[ct] }
             next if row_ids[key]
 
             row_ids[key] = true
           end
 
-          formatted_row = csv_column_names.map do |column|
-            if mapping[column][:type].to_sym == :datetime
-              # TODO: add some timezone config to the mapping.
-              # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
-              DateTime.parse(row[column]).utc rescue nil # rubocop:disable Style/RescueModifier
-            else
-              row[column]
-            end
-          end
-
           batcher << formatted_row
         }
 

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -40,7 +40,7 @@ module CanvasSync
       def insert_at(position, new_jobs)
         chain = self.class.get_chain_parameter(base_job)
         new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
-        chain.insert(-1, *new_jobs)
+        chain.insert(position, *new_jobs)
       end
 
       def insert(new_jobs, **kwargs)

data/lib/canvas_sync/jobs/report_checker.rb

@@ -4,8 +4,9 @@ module CanvasSync
     # Re-enqueues itself if the report is still processing on Canvas.
     # Enqueues the ReportProcessor when the report has completed.
     class ReportChecker < CanvasSync::Job
-      REPORT_TIMEOUT = 12.hours
+      REPORT_TIMEOUT = 24.hours
       COMPILATION_TIMEOUT = 1.hour
+      MAX_TRIES = 3
 
       # @param report_name [Hash] e.g., 'provisioning_csv'
       # @param report_id [Integer]
@@ -13,6 +14,7 @@ module CanvasSync
       # @param options [Hash] hash of options that will be passed to the job processor
       # @return [nil]
       def perform(report_name, report_id, processor, options, checker_context = {}) # rubocop:disable Metrics/AbcSize
+        max_tries = options[:report_max_tries] || batch_context[:report_max_tries] || MAX_TRIES
         account_id = options[:account_id] || batch_context[:account_id] || "self"
         report_status = CanvasSync.get_canvas_sync_client(batch_context)
                                   .report_status(account_id, report_name, report_id)
@@ -27,9 +29,17 @@ module CanvasSync
             report_id,
           )
         when "error", "deleted"
-          message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}" # rubocop:disable Metrics/LineLength
+          checker_context[:failed_attempts] ||= 0
+          checker_context[:failed_attempts] += 1
+          failed_attempts = checker_context[:failed_attempts]
+          message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}, #{current_organization.name}.  This report has now failed #{checker_context[:failed_attempts]} time." # rubocop:disable Metrics/LineLength
           Rails.logger.error(message)
-          raise message
+          if failed_attempts >= max_tries
+            Rails.logger.error("This report has failed #{failed_attempts} times.  Giving up.")
+            raise message
+          else
+            restart_report(options, report_name, processor, checker_context)
+          end
         else
           report_timeout = parse_timeout(options[:report_timeout] || batch_context[:report_timeout] || REPORT_TIMEOUT)
           if timeout_met?(options[:sync_start_time], report_timeout)
@@ -51,7 +61,7 @@ module CanvasSync
               report_id,
               processor,
               options,
-              checker_context,
+              checker_context
             )
         end
       end
@@ -66,6 +76,29 @@ module CanvasSync
       def parse_timeout(val)
         val
       end
+
+      def restart_report(options, report_name, processor, checker_context)
+        account_id = options[:account_id] || batch_context[:account_id] || "self"
+        options[:sync_start_time] = DateTime.now.utc.iso8601
+        new_context = {}
+        new_context[:failed_attempts] = checker_context[:failed_attempts]
+        report_id = start_report(account_id, report_name, options[:report_params])
+        CanvasSync::Jobs::ReportChecker
+            .set(wait: report_checker_wait_time)
+            .perform_later(
+              report_name,
+              report_id,
+              processor,
+              options,
+              new_context
+            )
+      end
+
+      def start_report(account_id, report_name, report_params)
+        report = CanvasSync.get_canvas_sync_client(batch_context)
+                           .start_report(account_id, report_name, report_params)
+        report["id"]
+      end
     end
   end
 end