canvas_sync 0.22.4 → 0.22.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b588205886b2732d915af7bd8a4e41fed60dae2593ca824c8fa046410258d6f3
-  data.tar.gz: 9f6a3238ae187f6a9a01135957ef8f769e127fc2870103c1bb7acacae57973ba
+  metadata.gz: 6fbc16dd889dae43881098a80d91c68ec64412b1ce872b51f176f89a0d2ce8fe
+  data.tar.gz: f202ff4cae0337f6a34872654b55371b5fdfabdccd1358fc7fceb14397549425
 SHA512:
-  metadata.gz: 9bbd02c072cd7f3eadce5f342b40eaa3f15f3a33843a0f93c20ff4afff0f401ae80532eddf99edd236027876bae20b0977cb0f5c10ddab57fb9d89f76c609efb
-  data.tar.gz: fa5845858716a91706e6b08877234635fced66701db9d6fc0e0c8956e3e1bb73e862cf7f69da062ba1823cd7cd8c1d23b66c97f20f51239523eaa8d59adcd9b8
+  metadata.gz: 35c21c3717c63c372fd840b511b2b51161f1114b4e9f8a8140d38ee0d4ca6393950d842b9619ded9731ae32b0abda8ef147ceaacba76f8bcc77d5b6966898b9f
+  data.tar.gz: a1b9a4f2129f9f441786237ed2bc69ff58009343d67d74f5724376f2fc77c0797fd3f932ea404d6b993d26ba210beeeafc1a9b0ad1025ad11169ebf7767f84af

data/README.md

@@ -242,6 +242,11 @@ class User < ApplicationRecord
     # If the column name on the report and in the DB are the same, a shorthand can be used:
     link_column :omit_from_final_grade, type: :datetime
 
+    # You can specify a block to pre-transform the value
+    link_column :column_in_report => :column_in_database do |value, row|
+      YAML.parse(value)
+    end
+
     # If the defaults define a column you don't want synced, you can remove it from the mapping:
     unlink_column :column_in_database
   end

data/app/models/canvas_sync/job_log.rb

@@ -3,7 +3,11 @@ module CanvasSync
   #
   # Use this model to track failures, job run times, and metadata about a job.
   class JobLog < ApplicationRecord
-    serialize :job_arguments, Array
+    if Rails.version > '7.0'
+      serialize :job_arguments, type: Array
+    else
+      serialize :job_arguments, Array
+    end
 
     ERROR_STATUS = "error".freeze
     SUCCESS_STATUS = "success".freeze

data/app/models/canvas_sync/sync_batch.rb

@@ -1,5 +1,4 @@
 module CanvasSync
   class SyncBatch < ApplicationRecord
-    serialize :job_arguments, Array
   end
 end

data/lib/canvas_sync/concerns/auto_relations.rb

@@ -0,0 +1,11 @@
+module CanvasSync::Concerns
+  module AutoRelations
+    extend ActiveSupport::Concern
+
+    # CanvasSync::Record.define_feature self, default: true
+
+    included do
+
+    end
+  end
+end

data/lib/canvas_sync/job_batches/batch.rb

@@ -478,20 +478,22 @@ module CanvasSync::JobBatches
 
       # Internal method to cleanup a Redis Hash and related keys
       def cleanup_redis_index_for(key, suffixes = [""])
-        if r.hget(k, "created_at").present?
-          r.multi do |r|
-            suffixes.each do |suffix|
-              r.expire(key + suffix, BID_EXPIRE_TTL)
+        redis do |r|
+          if r.hget(k, "created_at").present?
+            r.multi do |r|
+              suffixes.each do |suffix|
+                r.expire(key + suffix, BID_EXPIRE_TTL)
+              end
             end
-          end
-          false
-        else
-          r.multi do |r|
-            suffixes.each do |suffix|
-              r.unlink(key + suffix)
+            false
+          else
+            r.multi do |r|
+              suffixes.each do |suffix|
+                r.unlink(key + suffix)
+              end
             end
+            true
           end
-          true
         end
       end
 
@@ -499,14 +501,16 @@ module CanvasSync::JobBatches
       def cleanup_redis_index!
         suffixes = ["", "-callbacks-complete", "-callbacks-success", "-failed", "-dead", "-batches-success", "-batches-complete", "-batches-failed", "-bids", "-jids", "-pending_callbacks"]
 
-        cleanup_index = ->(index) {
-          r.zrangebyscore(index, "0", BID_EXPIRE_TTL.seconds.ago.to_i).each do |bid|
-            r.zrem(index, bid) if cleanup_redis_index_for("BID-#{bid}", suffixes)
-          end
-        }
+        redis do |r|
+          cleanup_index = ->(index) {
+            r.zrangebyscore(index, "0", BID_EXPIRE_TTL.seconds.ago.to_i).each do |bid|
+              r.zrem(index, bid) if cleanup_redis_index_for("BID-#{bid}", suffixes)
+            end
+          }
 
-        cleanup_index.call("BID-ROOT-bids")
-        cleanup_index.call("batches")
+          cleanup_index.("BID-ROOT-bids")
+          cleanup_index.("batches")
+        end
       end
 
       def redis(&blk)

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -4,7 +4,9 @@ module CanvasSync::JobBatches
 
     attr_reader :base_job
 
-    def initialize(base_type = SerialBatchJob)
+    def initialize(base_type = SerialBatchJob, chain_id: nil)
+      @chain_id = chain_id || SecureRandom.urlsafe_base64(10)
+
       if base_type.is_a?(Hash)
         @base_job = base_type
         @base_job[:args] ||= @base_job[:parameters] || []
@@ -109,7 +111,7 @@ module CanvasSync::JobBatches
       return nil if matching_jobs.count == 0
 
       job = matching_jobs[0][0]
-      job = self.class.new(job) unless job.is_a?(ChainBuilder)
+      job = self.class.new(job, chain_id: @chain_id) unless job.is_a?(ChainBuilder)
       job
     end
 
@@ -118,6 +120,7 @@ module CanvasSync::JobBatches
         job_def.normalize!
       else
         job_def[:job] = job_def[:job].to_s
+        job_def[:chain_link] ||= "#{@chain_id}-#{SecureRandom.urlsafe_base64(10)}"
         if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
           chain.map! { |sub_job| normalize!(sub_job) }
         end
@@ -138,7 +141,7 @@ module CanvasSync::JobBatches
         args: args,
         kwargs: kwargs,
       }
-      self.class.new(hsh).apply_block(&blk) if blk.present?
+      self.class.new(hsh, chain_id: @chain_id).apply_block(&blk) if blk.present?
       hsh
     end
 
@@ -239,6 +242,24 @@ module CanvasSync::JobBatches
           job_class.perform_later(*job_args, **job_kwargs)
         end
       end
+
+      def link_to_batch!(chain_link, batch)
+        # Or make chains a separate entity - Chains show batches, but batches don't show chain?
+        # Or "Annotate" a Batch with chain data - could extract chain id from any job entry
+      end
+
+      def annotate_batch!(batch, chain)
+
+      end
+
+      def handle_step_complete(status, opts)
+        chain_link = opts[:chain_link]
+        chain_id, chain_step_id = chain_link.split('-')
+
+        CanvasSync::JobBatches::Batch.redis.multi do |r|
+          r.hset("CHAIN-#{chain_id}-steps", chain_step_id, "complete")
+        end
+      end
     end
   end
 

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -144,6 +144,10 @@ module CanvasSync::JobBatches
             if next_job[:chain_link].present?
               # Annotate Batch with chain-step info
               batch.context["csb:chain_link"] = next_job[:chain_link]
+              # TODO Add Fiber Batch to chain-link
+              # With the exception of the top of the Chain, all nested ManagedBatch Roots should be within a Fiber,
+              #   so we shouldn't really need to make the Root checkin with the chain
+              #   ...except to cleanup the chain
               batch.on(:complete, "#{ChainBuilder.to_s}.chain_step_complete", chain_link: next_job[:chain_link])
             end
 

data/lib/canvas_sync/job_batches/status.rb

@@ -49,6 +49,7 @@ module CanvasSync::JobBatches
       end
 
       def success?
+        # TODO (Race Condition) This might not be valid if checked from a :complete callback
         'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'success') }
       end
 

data/lib/canvas_sync/job_uniqueness/lock_context.rb

@@ -6,8 +6,6 @@ module CanvasSync::JobUniqueness
       context_class.new(data, **kwargs)
     end
 
-    attr_reader :lock_id
-
     # { job_clazz, jid, queue, args?, kwargs?, base_key? }
     def initialize(data, job_instance: nil, config: nil)
       @base_key = data[:base_key]
@@ -16,7 +14,9 @@ module CanvasSync::JobUniqueness
       @config = config || @context_data[:config]
 
       # TODO Consider (somewhere) updating the lock_id to the BID of the wrapping Batch (when applicable)
-      @lock_id ||= data[:lid] || Thread.current[:unique_jobs_previous_context]&.lock_id || job_id
+      @lock_id ||= data[:lid] || Thread.current[:unique_jobs_previous_context]&.lock_id
+      @lock_id_locked = @lock_id.present?
+      @lock_id ||= job_id
     end
 
     # This is primarily for rehydrating in a Batch Callback, so it is unlikely that args and kwargs are needed.
@@ -84,6 +84,18 @@ module CanvasSync::JobUniqueness
       end
     end
 
+    def lock_id
+      @lock_id_locked = true
+      @lock_id
+    end
+
+    def lock_id=(new_id)
+      raise "Lock ID already set" if @lock_id_locked
+      return unless new_id.present?
+      @lock_id = new_id
+      @lock_id_locked = true
+    end
+
     def job_id
       @context_data[:jid]
     end
@@ -122,14 +134,27 @@ module CanvasSync::JobUniqueness
 
         args = @context_data[:args] || []
         kwargs = @context_data[:kwargs] || {}
+        hash = config[:hash]
         if config[:hash].is_a?(Proc)
-          base_key << config[:hash].call(*args, **kwargs)
+          hash = config[:hash].call(*args, **kwargs)
         elsif config[:hash].nil?
-          base_key << OpenSSL::Digest::MD5.hexdigest(JSON.dump([ args, kwargs.sort]))
-        else
-          base_key << config[:hash]
+          hash = [*args, kwargs]
+        end
+
+        hash = ":#{hash}" if hash.is_a?(Symbol)
+
+        if hash && !hash.is_a?(String)
+          hash = Array(hash)
+
+          # Normalize the hash to ensure that the order of any Hash keys don't matter
+          hash = normalize_hash_chunk(hash)
+
+          normalized = ActiveJob::Arguments.serialize(hash)
+          hash = OpenSSL::Digest::MD5.hexdigest(JSON.dump(normalized))
         end
 
+        base_key << hash if hash
+
         base_key.join(":")
       end
     end
@@ -173,5 +198,14 @@ module CanvasSync::JobUniqueness
 
     private
 
+    def normalize_hash_chunk(chunk)
+      if chunk.is_a?(Hash)
+        chunk.map { |k, v| [k, normalize_hash_chunk(v)] }.sort.to_h
+      elsif chunk.is_a?(Array)
+        chunk.map { |c| normalize_hash_chunk(c) }
+      else
+        chunk
+      end
+    end
   end
 end