canvas_sync 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68b09dd9ca869bb37f08c0691ce54f1efa4abb7e
-  data.tar.gz: 6615009014df4ddcd3b20e602549658fd659e0f9
+  metadata.gz: 450c9dd5440a29b40915b904cbbbe5eeeeff1c5d
+  data.tar.gz: 76035ff5b19e381ff046e6a57eec80943bb43254
 SHA512:
-  metadata.gz: c7526df95cf7d7cd2a86d48c95ca57b4da54d9a6e94095318b0eefc5ec08dc37112072e0be0246b655715b21a0c593a0d49fbfc6cf44a6b5ad647ee6aa57188d
-  data.tar.gz: 92b7dd1398562e07cacb45a5b5c54aac29ee4e366b090dbc5fec31e44690d4a211872c6dfe3d0b93c5aa71de5bff3f39da8f82d933f26484516a1fe875c17fd1
+  metadata.gz: eabefd522ae7150d7e9d5774440670fc647084ebad7786d3bd1b57dee0025d7a77607ffc923e16d76969986f29c6c47957042d3cf8873713676f284ac78d0aa0
+  data.tar.gz: 202029add66cc446630ba15c2a5816adbcdd2971085532d977e8cce40f365c5678d636833c9dd137b667bc18d27d25201d48da3b1ce7bb05f8c92b6003c8aee7

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -13,16 +13,25 @@ module CanvasSync
       # @param klass [Object] e.g., User
       # @param conflict_target [Symbol] represents the database column that will determine if we need to update
       #   or insert a given row. e.g.,: canvas_user_id
+      # @param exclude_duplicates [Boolean] importing will break if the file has any duplicate rows. Set this
+      #   to true in order to have the bulk importer filter those out.
       # @yieldparam [Array] row if a block is passed in it will yield the current row from the CSV.
       #   This can be used if you need to filter or massage the data in any way.
-      def self.import(report_file_path, mapping, klass, conflict_target)
+      def self.import(report_file_path, mapping, klass, conflict_target, exclude_duplicates=false)
         csv_column_names = mapping.keys
         database_column_names = mapping.values
         rows = []
+        row_ids = {}
 
         CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
           row = yield(row) if block_given?
           next if row.nil?
+
+          if exclude_duplicates
+            next if row_ids[row[conflict_target]]
+            row_ids[row[conflict_target]] = true
+          end
+
           rows << csv_column_names.map { |column| row[column] }
 
           if rows.length >= batch_size

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -92,16 +92,12 @@ module CanvasSync
       end
 
       def self.process_users(report_file_path)
-        rows = {}
-
-        # Users can show up more than once in a report if they have multiple logins. This breaks
-        # the bulk import, so the block we pass into the importer filters out users we've already
-        # encountered.
-        CanvasSync::Importers::BulkImporter.import(report_file_path, USERS_CSV_MAPPING, User, :canvas_user_id) do |row|
-          next nil if rows[row[:canvas_user_id]]
-          rows[row[:canvas_user_id]] = true
-          row
-        end
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          USERS_CSV_MAPPING,
+          User,
+          :canvas_user_id,
+          true)
       end
 
       def self.process_courses(report_file_path)
@@ -109,7 +105,12 @@ module CanvasSync
       end
 
       def self.process_enrollments(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(report_file_path, ENROLLMENTS_CSV_MAPPING, Enrollment, :canvas_enrollment_id)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          ENROLLMENTS_CSV_MAPPING,
+          Enrollment,
+          :canvas_enrollment_id,
+          true)
       end
 
       def self.process_sections(report_file_path)

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.1.7"
+  VERSION = "0.1.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-18 00:00:00.000000000 Z
+date: 2017-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -352,6 +352,7 @@ files:
 - spec/dummy/db/migrate/20170914181345_create_courses.rb
 - spec/dummy/db/migrate/20170918221413_create_users.rb
 - spec/dummy/db/schema.rb
+- spec/dummy/db/test.sqlite3
 - spec/dummy/log/development.log
 - spec/dummy/log/test.log
 - spec/factories/course_factory.rb
@@ -435,6 +436,7 @@ test_files:
 - spec/dummy/db/migrate/20170914181345_create_courses.rb
 - spec/dummy/db/migrate/20170918221413_create_users.rb
 - spec/dummy/db/schema.rb
+- spec/dummy/db/test.sqlite3
 - spec/dummy/log/development.log
 - spec/dummy/log/test.log
 - spec/factories/course_factory.rb