canvas_sync 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a91540c9a7c6e087d7b9c3f21340e689c0b77da
-  data.tar.gz: 9aad571654e1fe0b167f2af732bb1f49f4204a1f
+  metadata.gz: 6140a1f5a19c611e6ef92944993bcb0fb676e333
+  data.tar.gz: b26ca4c005bcdaa8fdfd4d6dc4f99342f391601a
 SHA512:
-  metadata.gz: 0654cbfc7984fca42f257a9ba0644fb21edf01b3d12d65ab53af9ca5698760c416168208162ab63b8fd98ce992c220ab26303218ed189f31b21d94539c5c0414
-  data.tar.gz: 9a256cb21575fb57108c39570a5259a967dfc4aa509e2bd48790e8f27aa35b72cc94f1cff7b15db877a72941e8ff7a1783b9635e1e7ecfb35f652e88f6ff6aca
+  metadata.gz: 2c27000087b00e839a5eb39fec4c03d08c193427ac19cfb8e9757f3103f5167a7f48fdd2f34af75f6257a6a8555d109f92494df2bf89b20e6c75ed6895b79b7b
+  data.tar.gz: 17987763dd6c6e7b47fe1815fa835cd57b4b1f01997879fd8f8279607b3f64b75b5c22eadeaaeeb8e3d0192235edcac69d7972a52e20aebab1a51a6842995700

data/app/models/canvas_sync/job_log.rb

@@ -4,5 +4,8 @@ module CanvasSync
   # Use this model to track failures, job run times, and metadata about a job.
   class JobLog < ApplicationRecord
     serialize :job_arguments, Array
+
+    ERROR_STATUS = 'error'
+    SUCCESS_STATUS = 'success'
   end
 end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -13,13 +13,13 @@ module CanvasSync
       # @param klass [Object] e.g., User
       # @param conflict_target [Symbol] represents the database column that will determine if we need to update
       #   or insert a given row. e.g.,: canvas_user_id
-      # @param exclude_duplicates [Boolean] importing will break if the file has any duplicate rows. Set this
-      #   to true in order to have the bulk importer filter those out.
+      # @param import_args [Hash] Any arguments passed here will be passed through to ActiveRecord::BulkImport.
+      #        Note: passing the key [:on_duplicate_key_ignore] will override the default behavior of [:on_duplicate_key_update]
       # @yieldparam [Array] row if a block is passed in it will yield the current row from the CSV.
       #   This can be used if you need to filter or massage the data in any way.
-      def self.import(report_file_path, mapping, klass, conflict_target, exclude_duplicates=false)
+      def self.import(report_file_path, mapping, klass, conflict_target, import_args: {})
         csv_column_names = mapping.keys
-        database_column_names = mapping.values.map{ |value| value[:database_column_name] }
+        database_column_names = mapping.values.map {|value| value[:database_column_name]}
         rows = []
         row_ids = {}
 
@@ -27,38 +27,41 @@ module CanvasSync
           row = yield(row) if block_given?
           next if row.nil?
 
-          if exclude_duplicates
-            next if row_ids[row[conflict_target]]
-            row_ids[row[conflict_target]] = true
-          end
+          next if row_ids[row[conflict_target]]
+          row_ids[row[conflict_target]] = true
 
           rows << csv_column_names.map do |column|
             if mapping[column][:type] == :datetime
-              DateTime.parse(row[column]).utc if row[column].present?
+              # todo - add some timezone config to the mapping.
+              # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
+              DateTime.parse(row[column]).utc rescue ''
             else
               row[column]
             end
           end
 
           if rows.length >= batch_size
-            perform_import(klass, database_column_names, rows, conflict_target)
+            perform_import(klass, database_column_names, rows, conflict_target, import_args)
             rows = []
+            row_ids = {}
           end
         end
 
-        perform_import(klass, database_column_names, rows, conflict_target)
+        perform_import(klass, database_column_names, rows, conflict_target, import_args)
       end
 
       private
 
-      def self.perform_import(klass, columns, rows, conflict_target)
+      def self.perform_import(klass, columns, rows, conflict_target, import_args={})
         return if rows.length == 0
         columns = columns.dup
-        klass.import(columns, rows, validate: false, on_duplicate_key_update: {
-          conflict_target: conflict_target,
-          condition: condition_sql(klass, columns),
-          columns: columns
-        })
+        options = {validate: false, on_duplicate_key_update: {
+            conflict_target: conflict_target,
+            condition: condition_sql(klass, columns),
+            columns: columns
+        }}.merge(import_args)
+        options.delete(:on_duplicate_key_update) if options.has_key?(:on_duplicate_key_ignore)
+        klass.import(columns, rows, options)
       end
 
       # This method generates SQL that looks like:
@@ -72,8 +75,8 @@ module CanvasSync
       # run_the_users_sync!
       # changed = User.where("updated_at >= ?", started_at)
       def self.condition_sql(klass, columns)
-        columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
-        excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
+        columns_str = columns.map {|c| "#{klass.quoted_table_name}.#{c}"}.join(", ")
+        excluded_str = columns.map {|c| "EXCLUDED.#{c}"}.join(", ")
         "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
       end
 

data/lib/canvas_sync/job.rb

@@ -5,9 +5,9 @@ module CanvasSync
   class Job < ActiveJob::Base
     around_perform do |job, block|
       @job_log = CanvasSync::JobLog.create(
-        started_at: Time.now,
-        job_class: self.class.name,
-        job_arguments: job.arguments
+          started_at: Time.now,
+          job_class: self.class.name,
+          job_arguments: job.arguments
       )
 
       begin
@@ -15,10 +15,16 @@ module CanvasSync
       rescue => e
         @job_log.exception = "#{e.class}: #{e.message}"
         @job_log.backtrace = e.backtrace
+        @job_log.status = JobLog::ERROR_STATUS
         raise e
       ensure
-        @job_log.completed_at = Time.now
-        @job_log.save!
+        if @job_log.job_class == 'CanvasSync::Jobs::ReportChecker' && @job_log.status != JobLog::ERROR_STATUS
+          @job_log.destroy
+        else
+          @job_log.completed_at = Time.now
+          @job_log.status ||= JobLog::SUCCESS_STATUS
+          @job_log.save!
+        end
       end
     end
 

data/lib/canvas_sync/jobs/report_processor_job.rb

@@ -13,6 +13,7 @@ module CanvasSync
       # @param options [Hash] hash of options that will be passed to the job processor
       # @return [nil]
       def perform(job_chain, report_name, report_url, processor, options)
+        @job_log.update_attributes(job_class: processor)
         download(report_name, report_url) do |file_path|
           options = options.merge({
             legacy_support: job_chain[:global_options][:legacy_support],

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -62,8 +62,7 @@ module CanvasSync
           report_file_path,
           mapping[:users][:report_columns],
           User,
-          mapping[:users][:conflict_target].to_sym,
-          true
+          mapping[:users][:conflict_target].to_sym
         )
       end
 
@@ -81,8 +80,7 @@ module CanvasSync
           report_file_path,
           mapping[:enrollments][:report_columns],
           Enrollment,
-          mapping[:enrollments][:conflict_target].to_sym,
-          true
+          mapping[:enrollments][:conflict_target].to_sym
         )
       end
 

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.3.0"
+  VERSION = '0.3.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-09 00:00:00.000000000 Z
+date: 2017-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler