canvas_sync 0.17.0.beta14 → 0.17.3.beta1

data/lib/canvas_sync/job_batches/redis_model.rb

@@ -0,0 +1,67 @@
+module CanvasSync
+  module JobBatches
+    module RedisModel
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def redis_attr(key, type = :string, read_only: true)
+          class_eval <<-RUBY, __FILE__, __LINE__ + 1
+            def #{key}=(value)
+              raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
+              @#{key} = value
+              if :#{type} == :json
+                value = JSON.unparse(value)
+              end
+              persist_bid_attr('#{key}', value)
+            end
+
+            def #{key}
+              return @#{key} if defined?(@#{key})
+              if (@initialized || @existing)
+                value = read_bid_attr('#{key}')
+                if :#{type} == :bool
+                  value = value == 'true'
+                elsif :#{type} == :int
+                  value = value.to_i
+                elsif :#{type} == :float
+                  value = value.to_f
+                elsif :#{type} == :json
+                  value = JSON.parse(value)
+                end
+                @#{key} = value
+              end
+            end
+          RUBY
+        end
+      end
+
+      def persist_bid_attr(attribute, value)
+        if @initialized || @existing
+          redis do |r|
+            r.multi do
+              r.hset(redis_key, attribute, value)
+              r.expire(redis_key, Batch::BID_EXPIRE_TTL)
+            end
+          end
+        else
+          @pending_attrs ||= {}
+          @pending_attrs[attribute] = value
+        end
+      end
+
+      def read_bid_attr(attribute)
+        redis do |r|
+          r.hget(redis_key, attribute)
+        end
+      end
+
+      def flush_pending_attrs
+        redis do |r|
+          r.mapped_hmset(redis_key, @pending_attrs)
+        end
+        @initialized = true
+        @pending_attrs = {}
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/redis_script.rb

@@ -0,0 +1,163 @@
+require 'pathname'
+require 'digest/sha1'
+require 'erb'
+
+# Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
+
+module CanvasSync
+  module JobBatches
+    # {RedisScript} represents a lua script in the filesystem. It loads the script
+    # from disk and handles talking to redis to execute it. Error handling
+    # is handled by {LuaError}.
+    class RedisScript
+
+      # Loads the script file from disk and calculates its +SHA1+ sum.
+      #
+      # @param file [Pathname] the full path to the indicated file
+      def initialize(file)
+        @file = Pathname.new(file)
+      end
+
+      # Passes the script and supplied arguments to redis for evaulation.
+      # It first attempts to use a script redis has already cached by using
+      # the +EVALSHA+ command, but falls back to providing the full script
+      # text via +EVAL+ if redis has not seen this script before. Future
+      # invocations will then use +EVALSHA+ without erroring.
+      #
+      # @param redis [Redis] the redis connection to run against
+      # @param args [*Objects] the arguments to the script
+      # @return [Object] the value passed back by redis after script execution
+      # @raise [LuaError] if the script failed to compile of encountered a
+      #   runtime error
+      def call(redis, *args)
+        t = Time.now
+        begin
+          redis.evalsha(digest, *args)
+        rescue => e
+          e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
+        end
+      rescue => e
+        if LuaError.intercepts?(e)
+          raise LuaError.new(e, @file, content)
+        else
+          raise
+        end
+      end
+
+      def content
+        @content ||= load_lua(@file)
+      end
+
+      def digest
+        @digest ||= Digest::SHA1.hexdigest content
+      end
+
+      private
+
+      def script_path
+        Rails.root + 'app/redis_lua'
+      end
+
+      def relative_path
+        @path ||= @file.relative_path_from(script_path)
+      end
+
+      def load_lua(file)
+        TemplateContext.new(script_path).template(script_path + file)
+      end
+
+      class TemplateContext
+        def initialize(script_path)
+          @script_path = script_path
+        end
+
+        def template(pathname)
+          @partial_templates ||= {}
+          ERB.new(File.read(pathname)).result binding
+        end
+
+        # helper method to include a lua partial within another lua script
+        #
+        # @param relative_path [String] the relative path to the script from
+        #     `script_path`
+        def include_partial(relative_path)
+          unless @partial_templates.has_key? relative_path
+            @partial_templates[relative_path] = nil
+            template( Pathname.new("#{@script_path}/#{relative_path}") )
+          end
+        end
+      end
+
+      # Reformats errors raised by redis representing failures while executing
+      # a lua script. The default errors have confusing messages and backtraces,
+      # and a type of +RuntimeError+. This class improves the message and
+      # modifies the backtrace to include the lua script itself in a reasonable
+      # way.
+      class LuaError < StandardError
+        PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
+        WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
+        CONTEXT_LINE_NUMBER = 2
+
+        attr_reader :error, :file, :content
+
+        # Is this error one that should be reformatted?
+        #
+        # @param error [StandardError] the original error raised by redis
+        # @return [Boolean] is this an error that should be reformatted?
+        def self.intercepts? error
+          error.message =~ PATTERN
+        end
+
+        # Initialize a new {LuaError} from an existing redis error, adjusting
+        # the message and backtrace in the process.
+        #
+        # @param error [StandardError] the original error raised by redis
+        # @param file [Pathname] full path to the lua file the error ocurred in
+        # @param content [String] lua file content the error ocurred in
+        def initialize error, file, content
+          @error = error
+          @file = file
+          @content = content
+
+          @error.message =~ PATTERN
+          _stage, line_number, message = $1, $2, $3
+          error_context = generate_error_context(content, line_number.to_i)
+
+          super "#{message}\n\n#{error_context}\n\n"
+          set_backtrace generate_backtrace file, line_number
+        end
+
+        private
+
+        def generate_error_context(content, line_number)
+          lines = content.lines.to_a
+          beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
+          ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
+          line_number_width = ending_line_number.to_s.length
+
+          (beginning_line_number..ending_line_number).map do |number|
+            indicator = number == line_number ? '=>' : '  '
+            formatted_number = "%#{line_number_width}d" % number
+            " #{indicator} #{formatted_number}: #{lines[number - 1]}"
+          end.join.chomp
+        end
+
+        def generate_backtrace(file, line_number)
+          pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
+          index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
+          pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
+          pre_wolverine.unshift("#{file}:#{line_number}")
+          pre_wolverine
+        end
+
+        def backtrace_before_entering_wolverine(backtrace)
+          backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
+        end
+
+        def line_from_wolverine(line)
+          line.split(':').first.include?(WOLVERINE_LIB_PATH)
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -15,11 +15,31 @@ module CanvasSync
           Thread.current[:batch]
         end
 
+        def batch_context
+          batch&.context || {}
+        end
+
         def valid_within_batch?
           batch.valid?
         end
       end
 
+      class SidekiqCallbackWorker
+        include ::Sidekiq::Worker
+        include WorkerExtension
+        include Batch::Callback::CallbackWorkerCommon
+
+        def self.enqueue_all(args, queue)
+          return if args.empty?
+
+          ::Sidekiq::Client.push_bulk(
+            'class' => self,
+            'args' => args,
+            'queue' => queue
+          )
+        end
+      end
+
       class ClientMiddleware
         def call(_worker, msg, _queue, _redis_pool = nil)
           if (batch = Thread.current[:batch]) && should_handle_batch?(msg)
@@ -29,7 +49,7 @@ module CanvasSync
         end
 
         def should_handle_batch?(msg)
-          return false if msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && msg['wrapped'].constantize.is_a?(BatchAwareJob)
+          return false if msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && msg['wrapped'].constantize < BatchAwareJob
           true
         end
       end
@@ -92,6 +112,7 @@ module CanvasSync
         # This alias helps apartment-sidekiq set itself up correctly
         ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
         ::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
+        Batch::Callback.worker_class = SidekiqCallbackWorker
       end
     end
   end

data/lib/canvas_sync/job_batches/status.rb

@@ -24,10 +24,6 @@ module CanvasSync
           Batch.redis { |r| r.hget("BID-#{bid}", 'created_at') }
         end
 
-        def total
-          Batch.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i
-        end
-
         def parent_bid
           Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
         end
@@ -63,7 +59,6 @@ module CanvasSync
         def data
           {
             bid: bid,
-            total: total,
             failures: failures,
             pending: pending,
             created_at: created_at,

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -6,7 +6,9 @@ module CanvasSync
       def perform(chain_definition, globals = {})
         @globals = globals
 
-        if !globals[:updated_after].present? || globals[:updated_after] == true
+        if globals[:updated_after] == false
+          globals[:updated_after] = nil
+        elsif !globals[:updated_after].present? || globals[:updated_after] == true
           last_batch = SyncBatch.where(status: 'completed', batch_genre: genre).last
           globals[:full_sync_every] ||= "sunday/2"
           globals[:updated_after] = last_batch&.started_at&.iso8601
@@ -39,7 +41,7 @@ module CanvasSync
         return true unless last_full_sync.present?
         return false unless opt.is_a?(String)
 
-        case r.strip
+        case opt.strip
         when %r{^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)(?:/(\d+))?$}
           m = Regexp.last_match
           day = m[1]

data/lib/canvas_sync/jobs/report_starter.rb

@@ -11,6 +11,7 @@ module CanvasSync
       # @return [nil]
       def perform(report_name, report_params, processor, options, allow_redownloads: false)
         account_id = options[:account_id] || batch_context[:account_id] || "self"
+        options[:sync_start_time] = DateTime.now.utc.iso8601
 
         report_id = start_report(account_id, report_name, report_params)
         # TODO: Restore report caching support (does nayone actually use it?)

data/lib/canvas_sync/processors/assignment_groups_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentGroupsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignment_groups][:report_columns],
           AssignmentGroup,
           mapping[:assignment_groups][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/assignments_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignments][:report_columns],
           Assignment,
           mapping[:assignments][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_module_items_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class ContextModuleItemsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_module_items][:report_columns],
           ContextModuleItem,
           mapping[:context_module_items][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_modules_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class ContextModulesProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_modules][:report_columns],
           ContextModule,
           mapping[:context_modules][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/normal_processor.rb

@@ -18,7 +18,8 @@ module CanvasSync
           report_file_path,
           mapping[options[:mapping].to_sym][:report_columns],
           options[:klass].constantize,
-          conflict_target ? conflict_target.to_sym : conflict_target
+          conflict_target ? conflict_target.to_sym : conflict_target,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -21,7 +21,6 @@ module CanvasSync
 
       def initialize(report_file_path, options) # rubocop:disable Metrics/AbcSize
         @options = options
-
         if options[:models].length == 1
           run_import(options[:models][0], report_file_path)
         else
@@ -75,6 +74,7 @@ module CanvasSync
           mapping[:users][:report_columns],
           User,
           mapping[:users][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -84,6 +84,7 @@ module CanvasSync
           mapping[:pseudonyms][:report_columns],
           Pseudonym,
           mapping[:pseudonyms][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -92,7 +93,8 @@ module CanvasSync
           report_file_path,
           mapping[:accounts][:report_columns],
           Account,
-          mapping[:accounts][:conflict_target].to_sym
+          mapping[:accounts][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -102,6 +104,7 @@ module CanvasSync
           mapping[:courses][:report_columns],
           Course,
           mapping[:courses][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -111,6 +114,7 @@ module CanvasSync
           mapping[:enrollments][:report_columns],
           Enrollment,
           mapping[:enrollments][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -120,6 +124,7 @@ module CanvasSync
           mapping[:sections][:report_columns],
           Section,
           mapping[:sections][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -129,6 +134,7 @@ module CanvasSync
           mapping[:xlist][:report_columns],
           Section,
           mapping[:xlist][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -138,6 +144,7 @@ module CanvasSync
           mapping[:groups][:report_columns],
           Group,
           mapping[:groups][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -148,6 +155,7 @@ module CanvasSync
           mapping[:group_memberships][:report_columns],
           GroupMembership,
           mapping[:group_memberships][:conflict_target].to_sym,
+          import_args: @options
         )
       end
     end