canvas_sync 0.22.5 → 0.22.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6fbc16dd889dae43881098a80d91c68ec64412b1ce872b51f176f89a0d2ce8fe
-  data.tar.gz: f202ff4cae0337f6a34872654b55371b5fdfabdccd1358fc7fceb14397549425
+  metadata.gz: '0882011929080b4e55e1798ae3639c4431a583dd2a31634b2b9c2ba5f450fcb4'
+  data.tar.gz: 7a339eef193de9b640b3d68d5474a7771d363f7dee723be49da33cf319afe610
 SHA512:
-  metadata.gz: 35c21c3717c63c372fd840b511b2b51161f1114b4e9f8a8140d38ee0d4ca6393950d842b9619ded9731ae32b0abda8ef147ceaacba76f8bcc77d5b6966898b9f
-  data.tar.gz: a1b9a4f2129f9f441786237ed2bc69ff58009343d67d74f5724376f2fc77c0797fd3f932ea404d6b993d26ba210beeeafc1a9b0ad1025ad11169ebf7767f84af
+  metadata.gz: a54b1779112dce3d106a503cc720b74d5b54ed2688c95c9d2c3523024813a0ac34d144417a8959bc81561dc4f78a7cf9914dd124037c85376a8a35b455bd5be1
+  data.tar.gz: 74e8b3e7d9317bbc2bbd7f902cfbedeabfd826ddbfd70aa4c8bf57c6f05dc2f4176f7bf2c0cc0352adae4ecaa4978f57f02fc3dcbbed8d1d8130df58bbaea51d

data/lib/canvas_sync/concerns/api_syncable.rb

@@ -40,14 +40,17 @@ module CanvasSync::Concerns
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
 
         if all_pages
-          batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
-            import(columns, batch, options)
+          enumer = api_array.all_pages_each.lazy
+
+          # Map the API response to instances of this model
+          enumer = enumer.map do |api_item|
+            new.assign_from_api_params(api_item)
           end
-          api_array.all_pages_each do |api_item|
-            item = new.assign_from_api_params(api_item)
-            batcher << item
+
+          # Import in batches
+          enumer.each_slice(batch_size) do |batch|
+            import(columns, batch, options)
           end
-          batcher.flush
         else
           items = api_array.map do |api_item|
             new.assign_from_api_params(api_item)

data/lib/canvas_sync/concerns/sync_mapping.rb

@@ -51,10 +51,11 @@ module CanvasSync::Concerns
             m = maps[mname] = {}
 
             m[:conflict_target] = Array(legacy[:conflict_target]).map(&:to_sym).map do |lct|
-              legacy[:report_columns][lct][:database_column_name]
+              legacy[:report_columns]&.[](lct)&.[](:database_column_name)
             end
 
             m[:columns] = {}
+
             legacy[:report_columns].each do |rcol, opts|
               m[:columns][opts[:database_column_name]] = opts.except(:database_column_name).merge!(
                 report_column: rcol,
@@ -68,6 +69,15 @@ module CanvasSync::Concerns
       def self.default_v1_mappings
         @legacy_mappings ||= begin
           mapping = YAML.load_file(File.join(__dir__, '../processors', "model_mappings.yml")).deep_symbolize_keys!
+          # Default columns can be excluded if the table has not been migrated to accept them
+          mapping.each do |mname, legacy|
+            legacy[:report_columns].select! do |rcol, opts|
+              model = mname.to_s&.classify&.safe_constantize
+              # we need to make sure this is a model otherwise they will be systematically removed - some mappings are not models (e.g xlists)
+              model.present? && !model.column_names.include?(opts[:database_column_name].to_s) ? false : true
+            end
+          end
+
           override_filepath = Rails.root.join("config/canvas_sync_provisioning_mapping.yml")
 
           if File.file?(override_filepath)

data/lib/canvas_sync/generators/templates/migrations/create_enrollments.rb

@@ -13,6 +13,7 @@ class CreateEnrollments < ActiveRecord::Migration[5.1]
       t.bigint :canvas_section_id
       t.string :workflow_state
       t.string :base_role_type
+      t.datetime :completed_at
 
       t.timestamps
     end

data/lib/canvas_sync/generators/templates/models/course_progress.rb

@@ -3,6 +3,7 @@
 # CourseProgress is not a Canvas model. It is a table built from the Custom Report
 class CourseProgress < ApplicationRecord
   include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
 
   canvas_sync_features :defaults
 
@@ -11,4 +12,11 @@ class CourseProgress < ApplicationRecord
 
   validates_presence_of :canvas_user_id, :canvas_course_id
   validates_uniqueness_of :canvas_user_id, scope: :canvas_course_id
+
+  api_syncable({
+    requirement_count: :requirement_count,
+    requirement_completed_count: :requirement_completed_count,
+    # provisioning report has completion_date instead of completed_at in the API
+    completion_date: :completed_at
+  }, -> (api) { api.course_progress(canvas_course_id, canvas_user_id) })
 end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -23,7 +23,7 @@ module CanvasSync
         end
       end
 
-      def self.perform_in_batches(report_file_path, raw_mapping, klass, conflict_target, import_args: {})
+      def self.perform_in_batches(report_file_path, raw_mapping, klass, conflict_target, import_args: {}, &block)
         mapping = {}.with_indifferent_access
         raw_mapping.each do |db_col, opts|
           next if opts[:deprecated] && !klass.column_names.include?(db_col.to_s)
@@ -37,14 +37,27 @@ module CanvasSync
         conflict_target = Array(conflict_target).map(&:to_s)
         conflict_target_indices = conflict_target.map{|ct| database_column_names.index(ct) }
 
-        row_ids = {}
-        batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
-          row_ids = {}
-          perform_import(klass, database_column_names, batch, conflict_target, import_args)
+        enumer = CSV.foreach(report_file_path, headers: true, header_converters: :symbol).lazy
+
+        # Optionally filter rows by a passed block
+        if block
+          enumer = enumer.filter_map do |row|
+            catch :skip do
+              block.call(row)
+            end
+          end
+        end
+
+        # Optionally chunk by a computed value. Mainly so we can group duplicate rows and choose one
+        chunker = nil
+        chunker = UserChunker.new if defined?(User) && klass == User && csv_column_names.include?('user_id')
+        if chunker
+          enumer = enumer.chunk{|row| chunker.key(row) }.flat_map{|key, rows| chunker.choose(key, rows) }
         end
 
-        row_buffer_out = ->(row) {
-          formatted_row = mapping.map do |db_col, col_def|
+        # Prepare the rows for import
+        enumer = enumer.map do |row|
+          mapping.map do |db_col, col_def|
             value = nil
             value = row[col_def[:report_column]] if col_def[:report_column]
 
@@ -63,33 +76,25 @@ module CanvasSync
 
             value
           end
+        end
 
-          if conflict_target.present?
-            key = conflict_target_indices.map{|ct| formatted_row[ct] }
-            next if row_ids[key]
-
+        # Reject rows within a single batch that have the same ID
+        row_ids = nil
+        if conflict_target.present?
+          enumer = enumer.reject do |row|
+            key = conflict_target_indices.map{|ct| row[ct] }
+            skip = row_ids[key]
             row_ids[key] = true
+            skip
           end
-
-          batcher << formatted_row
-        }
-
-        row_buffer = nil
-        if defined?(User) && klass == User && csv_column_names.include?('user_id')
-          row_buffer = UserRowBuffer.new(&row_buffer_out)
-        else
-          row_buffer = NullRowBuffer.new(&row_buffer_out)
         end
 
-        CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
-          row = yield(row) if block_given?
-          next if row.nil?
-
-          row_buffer << row
+        # Start importing
+        row_ids = {}
+        enumer.each_slice(batch_size) do |batch|
+          perform_import(klass, database_column_names, batch, conflict_target, import_args)
+          row_ids = {}
         end
-
-        row_buffer.flush
-        batcher.flush
       end
 
       def self.perform_import(klass, columns, rows, conflict_target, import_args={})
@@ -157,52 +162,17 @@ module CanvasSync
         batch_size > 0 ? batch_size : DEFAULT_BATCH_SIZE
       end
 
-      class RowBuffer
-        def initialize(&block)
-          @flush_out = block
-          @buffered_rows = []
-        end
-
-        def <<(v)
-          @buffered_rows << v
-        end
-
-        def flush(value = @buffered_rows)
-          if value.is_a?(Array)
-            value.each do |v|
-              @flush_out.call(v)
-            end
-          else
-            @flush_out.call(value)
-          end
-          @buffered_rows = []
-        end
-      end
-
-      class NullRowBuffer
-        def initialize(&block)
-          @flush_out = block
-        end
-
-        def <<(v)
-          @flush_out.call(v)
-        end
-
-        def flush; end
-      end
-
       # Ensures that, if a User has multiple rows, one with a SIS ID is preferred.
       # This is mainly to fix issues in legacy apps - the suggested approach for new apps
       # is to sync and use the Pseudonymes table
-      class UserRowBuffer < RowBuffer
-        def <<(v)
-          flush if @buffered_rows[0] && @buffered_rows[0][:canvas_user_id] != v[:canvas_user_id]
-          super
+      class UserChunker
+        def key(row)
+          row[:canvas_user_id]
         end
 
-        def flush
-          row = @buffered_rows.find{|r| r[:user_id].present? } || @buffered_rows.last
-          super(row.present? ? [row] : [])
+        def choose(key, rows)
+          row = rows.find{|r| r[:user_id].present? } || rows.last
+          row.present? ? [row] : []
         end
       end
     end

data/lib/canvas_sync/job_batches/batch.rb

@@ -337,7 +337,7 @@ module CanvasSync::JobBatches
           trigger_callback.call(:complete)
         end
 
-        if all_successful # All Successfull
+        if all_successful # All Successful
           trigger_callback.call(:success)
         elsif pending_jobs == dead_jobs && pending_batches == stagnated_batches # Stagnated
           trigger_callback.call(:stagnated)

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -4,9 +4,7 @@ module CanvasSync::JobBatches
 
     attr_reader :base_job
 
-    def initialize(base_type = SerialBatchJob, chain_id: nil)
-      @chain_id = chain_id || SecureRandom.urlsafe_base64(10)
-
+    def initialize(base_type = SerialBatchJob)
       if base_type.is_a?(Hash)
         @base_job = base_type
         @base_job[:args] ||= @base_job[:parameters] || []
@@ -111,7 +109,7 @@ module CanvasSync::JobBatches
       return nil if matching_jobs.count == 0
 
       job = matching_jobs[0][0]
-      job = self.class.new(job, chain_id: @chain_id) unless job.is_a?(ChainBuilder)
+      job = self.class.new(job) unless job.is_a?(ChainBuilder)
       job
     end
 
@@ -120,7 +118,6 @@ module CanvasSync::JobBatches
         job_def.normalize!
       else
         job_def[:job] = job_def[:job].to_s
-        job_def[:chain_link] ||= "#{@chain_id}-#{SecureRandom.urlsafe_base64(10)}"
         if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
           chain.map! { |sub_job| normalize!(sub_job) }
         end
@@ -141,7 +138,7 @@ module CanvasSync::JobBatches
         args: args,
         kwargs: kwargs,
       }
-      self.class.new(hsh, chain_id: @chain_id).apply_block(&blk) if blk.present?
+      self.class.new(hsh).apply_block(&blk) if blk.present?
       hsh
     end
 
@@ -242,24 +239,6 @@ module CanvasSync::JobBatches
           job_class.perform_later(*job_args, **job_kwargs)
         end
       end
-
-      def link_to_batch!(chain_link, batch)
-        # Or make chains a separate entity - Chains show batches, but batches don't show chain?
-        # Or "Annotate" a Batch with chain data - could extract chain id from any job entry
-      end
-
-      def annotate_batch!(batch, chain)
-
-      end
-
-      def handle_step_complete(status, opts)
-        chain_link = opts[:chain_link]
-        chain_id, chain_step_id = chain_link.split('-')
-
-        CanvasSync::JobBatches::Batch.redis.multi do |r|
-          r.hset("CHAIN-#{chain_id}-steps", chain_step_id, "complete")
-        end
-      end
     end
   end
 

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -144,10 +144,6 @@ module CanvasSync::JobBatches
             if next_job[:chain_link].present?
               # Annotate Batch with chain-step info
               batch.context["csb:chain_link"] = next_job[:chain_link]
-              # TODO Add Fiber Batch to chain-link
-              # With the exception of the top of the Chain, all nested ManagedBatch Roots should be within a Fiber,
-              #   so we shouldn't really need to make the Root checkin with the chain
-              #   ...except to cleanup the chain
               batch.on(:complete, "#{ChainBuilder.to_s}.chain_step_complete", chain_link: next_job[:chain_link])
             end
 

data/lib/canvas_sync/job_batches/status.rb

@@ -49,7 +49,6 @@ module CanvasSync::JobBatches
       end
 
       def success?
-        # TODO (Race Condition) This might not be valid if checked from a :complete callback
         'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'success') }
       end
 

data/lib/canvas_sync/job_uniqueness/lock_context.rb

@@ -6,6 +6,8 @@ module CanvasSync::JobUniqueness
       context_class.new(data, **kwargs)
     end
 
+    attr_reader :lock_id
+
     # { job_clazz, jid, queue, args?, kwargs?, base_key? }
     def initialize(data, job_instance: nil, config: nil)
       @base_key = data[:base_key]
@@ -14,9 +16,7 @@ module CanvasSync::JobUniqueness
       @config = config || @context_data[:config]
 
       # TODO Consider (somewhere) updating the lock_id to the BID of the wrapping Batch (when applicable)
-      @lock_id ||= data[:lid] || Thread.current[:unique_jobs_previous_context]&.lock_id
-      @lock_id_locked = @lock_id.present?
-      @lock_id ||= job_id
+      @lock_id ||= data[:lid] || Thread.current[:unique_jobs_previous_context]&.lock_id || job_id
     end
 
     # This is primarily for rehydrating in a Batch Callback, so it is unlikely that args and kwargs are needed.
@@ -84,18 +84,6 @@ module CanvasSync::JobUniqueness
       end
     end
 
-    def lock_id
-      @lock_id_locked = true
-      @lock_id
-    end
-
-    def lock_id=(new_id)
-      raise "Lock ID already set" if @lock_id_locked
-      return unless new_id.present?
-      @lock_id = new_id
-      @lock_id_locked = true
-    end
-
     def job_id
       @context_data[:jid]
     end

data/lib/canvas_sync/jobs/term_batches_job.rb

@@ -12,17 +12,14 @@ module CanvasSync
               # Override the delta-syncing date if:
               # 1. the Term hasn't been synced before or
               # 2. the Term underwent a period of not syncing
+              term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
               if batch_context[:updated_after]
-                term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
                 if !term_last_sync.present? || batch_context[:updated_after] > term_last_sync
                   local_context[:updated_after] = term_last_sync.presence
                 end
               end
 
               JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
-                # TODO If we do a Chain UI, this will need to checkin somehow to indicate that the chain forked
-                #   Or chain steps just show a summary - eg "Started", "X Jobs Running", "Done" or "X Jobs Running, Y Jobs Done" - and not individual forks
-                #   For a step to be considered done, all previous sibling-level steps must be done and no batches pending
                 b.description = "TermBatchJob(#{term_id}) Root"
                 b.context = local_context
                 b.on(:success, "#{self.class.to_s}.batch_finished") unless options[:mark_synced] == false

data/lib/canvas_sync/processors/model_mappings.yml

@@ -159,6 +159,9 @@ enrollments:
     base_role_type:
       database_column_name: base_role_type
       type: string
+    completed_at:
+      database_column_name: completed_at
+      type: datetime
 
 sections:
   conflict_target: canvas_section_id

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.22.5".freeze
+  VERSION = "0.22.8".freeze
 end