canvas_sync 0.22.6 → 0.22.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61d4f6b7e0c7f8201ee2dccdeea7e9f3149f112d35ea4ab72903ddbee892c079
-  data.tar.gz: 72a6a5553aa892680ee6ce2dacba9ce98d200cc44bd17a40dc5648b028b63868
+  metadata.gz: 0537b456eab7c4816125ddf995d80bb0afbd49a5d2f197fba4367e9753a85de7
+  data.tar.gz: e8789634293ef5517faba77d4c654036d45fc42f5aad69b836736dea2cd5f1dd
 SHA512:
-  metadata.gz: 5eb771935ecaa392a2fd6442136cadbe7972912381de6028c145de4e45419904de56a694a636dde1cabcbfda67fbeedfdcecd38993031194dab85f75ed529b98
-  data.tar.gz: fd8a0c64457a2190783527dcbfff67d34ec17f6f9249b1e0f4147a0e3ef8d5b9ae66ad4e132982b341ace536db78f39b95dbd5c7447f6099d765b83c8ef897ce
+  metadata.gz: 83fc5373c5bb541d163e4c5bcfee7adf43c2f5c70a6957a879d657b03a48a7ef5b96adde4b28fefadf913533942a538d7b014a0409ed41b6fed694191ec9d2ac
+  data.tar.gz: 5b212e20447e8a743e276407e7251ede63bdf1df2650d1378c60a40fedff6ef29efe716d25f1ec6a6c0ca1e7f6938c2b3b012d677189c5b158c876209ae5f48f

data/lib/canvas_sync/concerns/api_syncable.rb

@@ -40,14 +40,17 @@ module CanvasSync::Concerns
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
 
         if all_pages
-          batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
-            import(columns, batch, options)
+          enumer = api_array.all_pages_each.lazy
+
+          # Map the API response to instances of this model
+          enumer = enumer.map do |api_item|
+            new.assign_from_api_params(api_item)
           end
-          api_array.all_pages_each do |api_item|
-            item = new.assign_from_api_params(api_item)
-            batcher << item
+
+          # Import in batches
+          enumer.each_slice(batch_size) do |batch|
+            import(columns, batch, options)
           end
-          batcher.flush
         else
           items = api_array.map do |api_item|
             new.assign_from_api_params(api_item)

data/lib/canvas_sync/concerns/sync_mapping.rb

@@ -51,10 +51,11 @@ module CanvasSync::Concerns
             m = maps[mname] = {}
 
             m[:conflict_target] = Array(legacy[:conflict_target]).map(&:to_sym).map do |lct|
-              legacy[:report_columns][lct][:database_column_name]
+              legacy[:report_columns]&.[](lct)&.[](:database_column_name)
             end
 
             m[:columns] = {}
+
             legacy[:report_columns].each do |rcol, opts|
               m[:columns][opts[:database_column_name]] = opts.except(:database_column_name).merge!(
                 report_column: rcol,
@@ -68,6 +69,15 @@ module CanvasSync::Concerns
       def self.default_v1_mappings
         @legacy_mappings ||= begin
           mapping = YAML.load_file(File.join(__dir__, '../processors', "model_mappings.yml")).deep_symbolize_keys!
+          # Default columns can be excluded if the table has not been migrated to accept them
+          mapping.each do |mname, legacy|
+            legacy[:report_columns].select! do |rcol, opts|
+              model = mname.to_s&.classify&.safe_constantize
+              # we need to make sure this is a model otherwise they will be systematically removed - some mappings are not models (e.g xlists)
+              model.present? && !model.column_names.include?(opts[:database_column_name].to_s) ? false : true
+            end
+          end
+
           override_filepath = Rails.root.join("config/canvas_sync_provisioning_mapping.yml")
 
           if File.file?(override_filepath)

data/lib/canvas_sync/generators/templates/migrations/create_assignment_overrides.rb

@@ -0,0 +1,25 @@
+class CreateAssignmentOverrides < ActiveRecord::Migration[5.1]
+  def change
+    create_table :assignment_overrides do |t|
+      t.bigint :canvas_id, null: false
+      t.bigint :canvas_assignment_id
+      t.bigint :canvas_group_id
+      t.bigint :canvas_course_id
+      t.bigint :canvas_section_id
+      t.bigint :student_ids, array: true
+      t.datetime :due_at
+      t.boolean :all_day
+      t.datetime :all_day_date
+      t.datetime :unlock_at
+      t.datetime :lock_at
+      t.string :title
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :assignment_overrides, :canvas_id, unique: true
+    add_index :assignment_overrides, :canvas_assignment_id
+    add_index :assignment_overrides, :canvas_course_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_enrollments.rb

@@ -13,6 +13,7 @@ class CreateEnrollments < ActiveRecord::Migration[5.1]
       t.bigint :canvas_section_id
       t.string :workflow_state
       t.string :base_role_type
+      t.datetime :completed_at
 
       t.timestamps
     end

data/lib/canvas_sync/generators/templates/models/assignment_override.rb

@@ -0,0 +1,28 @@
+# <%= autogenerated_model_warning %>
+
+class AssignmentOverride < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  canvas_sync_features :defaults
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :assignment, primary_key: :canvas_id, foreign_key: :canvas_assignment_id, optional: true
+  belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
+
+  api_syncable({
+    canvas_id: :id,
+    canvas_assignment_id: :assignment_id,
+    title: :title,
+    workflow_state: :workflow_state,
+    due_at: :due_at,
+    all_day: :all_day,
+    all_day_date: :all_day_date,
+    unlock_at: :unlock_at,
+    lock_at: :lock_at,
+    student_ids: ->(r) { JSON.parse(r[:student_ids]) if r[:student_ids].present? },
+    canvas_group_id: :group_id,
+    canvas_section_id: :course_section_id,
+    canvas_course_id: -> (r) { r.dig(:assignment, :course_id) }
+  }, -> (api) { api.assignment_override(canvas_assignment_id, canvas_id) })
+end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -23,7 +23,7 @@ module CanvasSync
         end
       end
 
-      def self.perform_in_batches(report_file_path, raw_mapping, klass, conflict_target, import_args: {})
+      def self.perform_in_batches(report_file_path, raw_mapping, klass, conflict_target, import_args: {}, &block)
         mapping = {}.with_indifferent_access
         raw_mapping.each do |db_col, opts|
           next if opts[:deprecated] && !klass.column_names.include?(db_col.to_s)
@@ -37,14 +37,27 @@ module CanvasSync
         conflict_target = Array(conflict_target).map(&:to_s)
         conflict_target_indices = conflict_target.map{|ct| database_column_names.index(ct) }
 
-        row_ids = {}
-        batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
-          row_ids = {}
-          perform_import(klass, database_column_names, batch, conflict_target, import_args)
+        enumer = CSV.foreach(report_file_path, headers: true, header_converters: :symbol).lazy
+
+        # Optionally filter rows by a passed block
+        if block
+          enumer = enumer.filter_map do |row|
+            catch :skip do
+              block.call(row)
+            end
+          end
+        end
+
+        # Optionally chunk by a computed value. Mainly so we can group duplicate rows and choose one
+        chunker = nil
+        chunker = UserChunker.new if defined?(User) && klass == User && csv_column_names.include?('user_id')
+        if chunker
+          enumer = enumer.chunk{|row| chunker.key(row) }.flat_map{|key, rows| chunker.choose(key, rows) }
         end
 
-        row_buffer_out = ->(row) {
-          formatted_row = mapping.map do |db_col, col_def|
+        # Prepare the rows for import
+        enumer = enumer.map do |row|
+          mapping.map do |db_col, col_def|
             value = nil
             value = row[col_def[:report_column]] if col_def[:report_column]
 
@@ -63,33 +76,25 @@ module CanvasSync
 
             value
           end
+        end
 
-          if conflict_target.present?
-            key = conflict_target_indices.map{|ct| formatted_row[ct] }
-            next if row_ids[key]
-
+        # Reject rows within a single batch that have the same ID
+        row_ids = nil
+        if conflict_target.present?
+          enumer = enumer.reject do |row|
+            key = conflict_target_indices.map{|ct| row[ct] }
+            skip = row_ids[key]
             row_ids[key] = true
+            skip
           end
-
-          batcher << formatted_row
-        }
-
-        row_buffer = nil
-        if defined?(User) && klass == User && csv_column_names.include?('user_id')
-          row_buffer = UserRowBuffer.new(&row_buffer_out)
-        else
-          row_buffer = NullRowBuffer.new(&row_buffer_out)
         end
 
-        CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
-          row = yield(row) if block_given?
-          next if row.nil?
-
-          row_buffer << row
+        # Start importing
+        row_ids = {}
+        enumer.each_slice(batch_size) do |batch|
+          perform_import(klass, database_column_names, batch, conflict_target, import_args)
+          row_ids = {}
         end
-
-        row_buffer.flush
-        batcher.flush
       end
 
       def self.perform_import(klass, columns, rows, conflict_target, import_args={})
@@ -157,52 +162,17 @@ module CanvasSync
         batch_size > 0 ? batch_size : DEFAULT_BATCH_SIZE
       end
 
-      class RowBuffer
-        def initialize(&block)
-          @flush_out = block
-          @buffered_rows = []
-        end
-
-        def <<(v)
-          @buffered_rows << v
-        end
-
-        def flush(value = @buffered_rows)
-          if value.is_a?(Array)
-            value.each do |v|
-              @flush_out.call(v)
-            end
-          else
-            @flush_out.call(value)
-          end
-          @buffered_rows = []
-        end
-      end
-
-      class NullRowBuffer
-        def initialize(&block)
-          @flush_out = block
-        end
-
-        def <<(v)
-          @flush_out.call(v)
-        end
-
-        def flush; end
-      end
-
       # Ensures that, if a User has multiple rows, one with a SIS ID is preferred.
       # This is mainly to fix issues in legacy apps - the suggested approach for new apps
       # is to sync and use the Pseudonymes table
-      class UserRowBuffer < RowBuffer
-        def <<(v)
-          flush if @buffered_rows[0] && @buffered_rows[0][:canvas_user_id] != v[:canvas_user_id]
-          super
+      class UserChunker
+        def key(row)
+          row[:canvas_user_id]
         end
 
-        def flush
-          row = @buffered_rows.find{|r| r[:user_id].present? } || @buffered_rows.last
-          super(row.present? ? [row] : [])
+        def choose(key, rows)
+          row = rows.find{|r| r[:user_id].present? } || rows.last
+          row.present? ? [row] : []
         end
       end
     end

data/lib/canvas_sync/job_batches/batch.rb

@@ -337,7 +337,7 @@ module CanvasSync::JobBatches
           trigger_callback.call(:complete)
         end
 
-        if all_successful # All Successfull
+        if all_successful # All Successful
           trigger_callback.call(:success)
         elsif pending_jobs == dead_jobs && pending_batches == stagnated_batches # Stagnated
           trigger_callback.call(:stagnated)

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -4,9 +4,7 @@ module CanvasSync::JobBatches
 
     attr_reader :base_job
 
-    def initialize(base_type = SerialBatchJob, chain_id: nil)
-      @chain_id = chain_id || SecureRandom.urlsafe_base64(10)
-
+    def initialize(base_type = SerialBatchJob)
       if base_type.is_a?(Hash)
         @base_job = base_type
         @base_job[:args] ||= @base_job[:parameters] || []
@@ -111,7 +109,7 @@ module CanvasSync::JobBatches
       return nil if matching_jobs.count == 0
 
       job = matching_jobs[0][0]
-      job = self.class.new(job, chain_id: @chain_id) unless job.is_a?(ChainBuilder)
+      job = self.class.new(job) unless job.is_a?(ChainBuilder)
       job
     end
 
@@ -120,7 +118,6 @@ module CanvasSync::JobBatches
         job_def.normalize!
       else
         job_def[:job] = job_def[:job].to_s
-        job_def[:chain_link] ||= "#{@chain_id}-#{SecureRandom.urlsafe_base64(10)}"
         if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
           chain.map! { |sub_job| normalize!(sub_job) }
         end
@@ -141,7 +138,7 @@ module CanvasSync::JobBatches
         args: args,
         kwargs: kwargs,
       }
-      self.class.new(hsh, chain_id: @chain_id).apply_block(&blk) if blk.present?
+      self.class.new(hsh).apply_block(&blk) if blk.present?
       hsh
     end
 
@@ -242,24 +239,6 @@ module CanvasSync::JobBatches
           job_class.perform_later(*job_args, **job_kwargs)
         end
       end
-
-      def link_to_batch!(chain_link, batch)
-        # Or make chains a separate entity - Chains show batches, but batches don't show chain?
-        # Or "Annotate" a Batch with chain data - could extract chain id from any job entry
-      end
-
-      def annotate_batch!(batch, chain)
-
-      end
-
-      def handle_step_complete(status, opts)
-        chain_link = opts[:chain_link]
-        chain_id, chain_step_id = chain_link.split('-')
-
-        CanvasSync::JobBatches::Batch.redis.multi do |r|
-          r.hset("CHAIN-#{chain_id}-steps", chain_step_id, "complete")
-        end
-      end
     end
   end