canvas_sync 0.1.8 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 450c9dd5440a29b40915b904cbbbe5eeeeff1c5d
-  data.tar.gz: 76035ff5b19e381ff046e6a57eec80943bb43254
+  metadata.gz: 73fb470e4beeaee615dcf4cdb7170c973a2f9edb
+  data.tar.gz: df158059b78f81df04576230dee55e6bc230b755
 SHA512:
-  metadata.gz: eabefd522ae7150d7e9d5774440670fc647084ebad7786d3bd1b57dee0025d7a77607ffc923e16d76969986f29c6c47957042d3cf8873713676f284ac78d0aa0
-  data.tar.gz: 202029add66cc446630ba15c2a5816adbcdd2971085532d977e8cce40f365c5678d636833c9dd137b667bc18d27d25201d48da3b1ce7bb05f8c92b6003c8aee7
+  metadata.gz: 2af3c510be69d2432e57f21789b2cb2ca3dcdc61c7fa6b3b3d7d31fb67b00d4e45527bc0ca2eb9ee73c12764581676ca8b1cd2faf2adbae12f8155d404692c25
+  data.tar.gz: 174b2272acc70ffa36f2be27192fa366afcb9f18940bafda33d4ba8d8780ea24bdd72663ea1a28ece5dbc5dd88e8f2786e7baaab3d670ff7c0e8d964fd6954ad

data/README.md

@@ -54,13 +54,13 @@ end
 Once that's done and you've used the generator to create your models and migrations you can run the standard provisioning sync:
 
 ```ruby
-CanvasSync.provisioning_sync(<array of models to sync>, <optional term scope>)
+CanvasSync.provisioning_sync(<array of models to sync>, term_scope: <optional term scope>)
 ```
 
 Example:
 
 ```ruby
-CanvasSync.provisioning_sync(['users', 'courses'], :active)
+CanvasSync.provisioning_sync(['users', 'courses'], term_scope: :active)
 ```
 
 This will kick off a string of jobs to sync your specified models.
@@ -69,6 +69,16 @@ If you pass in the optional `term_scope` the provisioning reports will be run fo
 
 Imports are inserted in bulk with [https://github.com/zdennis/activerecord-import](activerecord-import) so they should be very fast.
 
+## Legacy Support
+
+If you have an old style tool that needs to sync data on a row by row basis, you can pass in the `legacy_support: true` option. In order for this to work, your models must have a `create_or_update_from_csv` class method defined that accepts a row argument. This method will get passed each row from the CSV, and it's up to you to persist it.
+
+Example:
+
+```ruby
+CanvasSync.provisioning_sync(['users', 'courses'], term_scope: :active, legacy_support: true)
+```
+
 ## CanvasSync::JobLog
 
 Running the migrations will create a `canvas_sync_job_logs` table. All the jobs written in this gem will create a `CanvasSync::JobLog` and store data about their arguments, job class, any exceptions, and start/completion time. This will work regardless of your queue adapter.
@@ -200,7 +210,7 @@ When adding to or updating this gem, make sure you do the following:
 
 ## TODO
 
-Currently in a pre-alpha like state.
-
-- Add better support for parallel job execution with job dependencies
-- Add some sort of `JobLog` model for tracking parallel execution and job metadata
+- Rethink how options are passed around. The current strategy of having "global_options" and per job options works
+  decently, but can be confusing. The difficulty is representing the options in a way that is easily serializable
+  by the queue adaptor and easily passed around.
+- Add support for defining "mappings" if your models don't match up with what CanvasSync expects.

data/db/migrate/20170915210836_create_canvas_sync_job_log.rb

@@ -1,4 +1,4 @@
-class CreateCanvasSyncJobLog < ActiveRecord::Migration[5.1]
+class CreateCanvasSyncJobLog < ActiveRecord::Migration[Rails.version.to_f]
   def change
     create_table :canvas_sync_job_logs do |t|
       t.datetime :started_at

data/lib/canvas_sync.rb

@@ -26,9 +26,14 @@ module CanvasSync
   #   must be one of SUPPORTED_MODELS
   # @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model.
   #   The provisioning report will be run for each of the terms contained in that scope.
-  def self.provisioning_sync(models, term_scope=nil)
+  # @param legacy_support [Boolean, false] This enables legacy_support, where rows are not bulk inserted.
+  #   For this to work your models must have a `create_or_udpate_from_csv` class method that takes a row
+  #   and inserts it into the database.
+  # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
+  #   canvas_sync_client methods require an account ID.
+  def self.provisioning_sync(models, term_scope: nil, legacy_support: false, account_id: nil)
     validate_models!(models)
-    invoke_next(default_provisioning_report_chain(models, term_scope))
+    invoke_next(default_provisioning_report_chain(models, term_scope, legacy_support, account_id))
   end
 
   # Runs a chain of ordered jobs
@@ -62,10 +67,16 @@ module CanvasSync
   #
   # @param models [Array<String>]
   # @param term_scope [String]
+  # @param legacy_support [Boolean, false] This enables legacy_support, where rows are not bulk inserted.
+  #   For this to work your models must have a `create_or_udpate_from_csv` class method that takes a row
+  #   and inserts it into the database.
+  # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
+  #   canvas_sync_client methods require an account ID.
   # @return [Hash]
-  def self.default_provisioning_report_chain(models, term_scope=nil)
+  def self.default_provisioning_report_chain(models, term_scope=nil, legacy_support=false, account_id=nil)
     term_scope = term_scope.to_s if term_scope
 
+
     # Always sync Terms first
     jobs = [{ job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} }]
 
@@ -77,7 +88,22 @@ module CanvasSync
 
     jobs.push({ job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: term_scope, models: models } })
 
-    { jobs: jobs, options: {} }
+    global_options = { legacy_support: legacy_support }
+    global_options[:account_id] = account_id if account_id.present?
+
+    { jobs: jobs, global_options: global_options }
+  end
+
+  # Calls the canvas_sync_client in your app. If you have specified an account
+  # ID when starting the job it will pass the account ID to your canvas_sync_client method.
+  #
+  # @param options [Hash]
+  def self.get_canvas_sync_client(options)
+    if options[:account_id]
+      canvas_sync_client(options[:account_id])
+    else
+      canvas_sync_client
+    end
   end
 
   private

data/lib/canvas_sync/importers/legacy_importer.rb

@@ -0,0 +1,17 @@
+module CanvasSync
+  module Importers
+    class LegacyImporter
+      # Does a legacy style import, row by row. This will be invoked if you passed
+      # "legacy_support: true" when starting the sync. For this to work your models
+      # must have a `create_or_update_from_csv` class method defined.
+      #
+      # @param report_file_path [String]
+      # @param klass [Object]
+      def self.import(report_file_path, klass, account_id)
+        CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
+          klass.create_or_update_from_csv(row, account_id)
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/report_checker.rb

@@ -11,7 +11,7 @@ module CanvasSync
       # @param options [Hash] hash of options that will be passed to the job processor
       # @return [nil]
       def perform(job_chain, report_name, report_id, processor, options)
-        report_status = canvas_sync_client.report_status('self', report_name, report_id)
+        report_status = CanvasSync.get_canvas_sync_client(job_chain[:global_options]).report_status('self', report_name, report_id)
 
         case report_status['status'].downcase
           when 'complete'

data/lib/canvas_sync/jobs/report_processor_job.rb

@@ -14,6 +14,10 @@ module CanvasSync
       # @return [nil]
       def perform(job_chain, report_name, report_url, processor, options)
         download(report_name, report_url) do |file_path|
+          options = options.merge({
+            legacy_support: job_chain[:global_options][:legacy_support],
+            account_id: job_chain[:global_options][:account_id]
+          })
           processor.constantize.process(file_path, options)
         end
 

data/lib/canvas_sync/jobs/report_starter.rb

@@ -9,7 +9,7 @@ module CanvasSync
       # @param options [Hash] hash of options that will be passed to the job processor
       # @return [nil]
       def perform(job_chain, report_name, report_params, processor, options)
-        report = canvas_sync_client.start_report('self', report_name, report_params)
+        report = CanvasSync.get_canvas_sync_client(job_chain[:global_options]).start_report('self', report_name, report_params)
 
         CanvasSync::Jobs::ReportChecker.set(wait: report_checker_wait_time).perform_later(
           job_chain,

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -12,7 +12,7 @@ module CanvasSync
             # Deep copy the job_chain so each report gets the correct term id passed into
             # its options with no side effects
             duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-            duped_job_chain[:options][:canvas_term_id] = term.canvas_term_id
+            duped_job_chain[:global_options][:canvas_term_id] = term.canvas_term_id
             start_report(report_params(options, term.canvas_term_id), duped_job_chain, options)
           end
         else

data/lib/canvas_sync/jobs/sync_terms_job.rb

@@ -8,8 +8,12 @@ module CanvasSync
       # @param job_chain [Hash]
       # @param options [Hash]
       def perform(job_chain, options)
-        canvas_sync_client.terms('self').all_pages!.each do |term_params|
-          Term.create_or_update(term_params)
+        CanvasSync.get_canvas_sync_client(job_chain[:global_options]).terms('self').all_pages!.each do |term_params|
+          if(job_chain[:global_options][:account_id])
+            Term.create_or_update(term_params, job_chain[:global_options][:account_id])
+          else
+            Term.create_or_update(term_params)
+          end
         end
 
         CanvasSync.invoke_next(job_chain)

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -64,20 +64,26 @@ module CanvasSync
       # @param report_file_path [String]
       # @param options [Hash]
       def self.process(report_file_path, options)
+        self.new(report_file_path, options)
+      end
+
+      def initialize(report_file_path, options)
+        @options = options
+
         if options[:models].length == 1
-          send("process_#{options[:models][0]}", report_file_path)
+          run_import(options[:models][0], report_file_path)
         else
           unzipped_file_path = extract(report_file_path)
           Dir[unzipped_file_path + "/*.csv"].each do |file_path|
             model_name = file_path.split("/").last.split(".").first
-            send("process_#{model_name}", file_path)
+            run_import(model_name, file_path)
           end
         end
       end
 
       private
 
-      def self.extract(file_path)
+      def extract(file_path)
         unzipped_file_path = "#{file_path}_unzipped"
 
         Zip::File.open(file_path) do |zip_file|
@@ -91,30 +97,50 @@ module CanvasSync
         unzipped_file_path
       end
 
-      def self.process_users(report_file_path)
+      def run_import(model_name, report_file_path)
+        if @options[:legacy_support]
+          CanvasSync::Importers::LegacyImporter.import(report_file_path, model_name.singularize.capitalize.constantize, @options[:account_id])
+        else
+          send("bulk_process_#{model_name}", report_file_path)
+        end
+      end
+
+      def bulk_process_users(report_file_path)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           USERS_CSV_MAPPING,
           User,
           :canvas_user_id,
-          true)
+          true
+        )
       end
 
-      def self.process_courses(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(report_file_path, COURSES_CSV_MAPPING, Course, :canvas_course_id)
+      def bulk_process_courses(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          COURSES_CSV_MAPPING,
+          Course,
+          :canvas_course_id
+        )
       end
 
-      def self.process_enrollments(report_file_path)
+      def bulk_process_enrollments(report_file_path)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           ENROLLMENTS_CSV_MAPPING,
           Enrollment,
           :canvas_enrollment_id,
-          true)
+          true
+        )
       end
 
-      def self.process_sections(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(report_file_path, SECTIONS_CSV_MAPPING, Section, :canvas_section_id)
+      def bulk_process_sections(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          SECTIONS_CSV_MAPPING,
+          Section,
+          :canvas_section_id
+        )
       end
     end
   end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.1.8"
+  VERSION = "0.2.0"
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -33,7 +33,7 @@ RSpec.describe CanvasSync do
               { job: CanvasSync::Jobs::SyncUsersJob.to_s, options: {} },
               { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
             ],
-            options: {}
+            global_options: { legacy_support: false }
           })
         end
       end
@@ -46,7 +46,7 @@ RSpec.describe CanvasSync do
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
               { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: ['users', 'courses'] } }
             ],
-            options: {}
+            global_options: { legacy_support: false }
           })
         end
       end

data/spec/canvas_sync/jobs/report_checker_spec.rb

@@ -5,11 +5,12 @@ RSpec.describe CanvasSync::Jobs::ReportChecker do
   let(:report_id) { 1 }
   let(:report_name) { 'provisioning_csv' }
   let(:processor) { 'FakeProcessor' }
+  let(:job_chain) { { jobs: [], global_options: {} } }
 
   describe '#perform' do
     def start_job
       CanvasSync::Jobs::ReportChecker.perform_now(
-        [],
+        job_chain,
         'provisioning_csv',
         report_id,
         processor,
@@ -24,7 +25,7 @@ RSpec.describe CanvasSync::Jobs::ReportChecker do
           .and_return({ 'status' => 'complete', 'attachment' => { 'url' => 'blah' } })
 
         expect(CanvasSync::Jobs::ReportProcessorJob).to receive(:perform_later)
-          .with([], report_name, 'blah', processor, {})
+          .with(job_chain, report_name, 'blah', processor, {})
 
         start_job
       end

data/spec/canvas_sync/jobs/report_processor_job_spec.rb

@@ -6,7 +6,7 @@ RSpec.describe CanvasSync::Jobs::ReportProcessorJob do
   let(:report_name) { 'provisioning_csv' }
   let(:report_url) { 'https://test.instructure.com/sample_report_download' }
   let(:processor) { FakeProcessor.to_s }
-  let(:job_chain) { ['another-job'] }
+  let(:job_chain) { { jobs: [], global_options: {} } }
 
   describe '#perform' do
     it 'downloads the report to a file and then calls the process method on the processor, and then invokes the next job' do

data/spec/canvas_sync/jobs/report_starter_spec.rb

@@ -14,7 +14,7 @@ RSpec.describe CanvasSync::Jobs::ReportStarter do
       expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original
 
       CanvasSync::Jobs::ReportStarter.perform_now(
-                                [],
+                                { jobs: [], global_options: {} },
                                 report_name,
                                 report_params,
                                 processor,

data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 RSpec.describe CanvasSync::Jobs::SyncProvisioningReportJob do
   describe '#perform' do
-    let(:job_chain) { {jobs: [], options: {}} }
+    let(:job_chain) { {jobs: [], global_options: {}} }
 
     context 'a term scope is specified' do
       let!(:active_term_1) { FactoryGirl.create(:term) }
@@ -11,7 +11,7 @@ RSpec.describe CanvasSync::Jobs::SyncProvisioningReportJob do
 
       it 'enqueues a ReportStarter for a provisioning report for the specified models for each term' do
         expected_job_chain = Marshal.load(Marshal.dump(job_chain))
-        expected_job_chain[:options][:canvas_term_id] = active_term_1.canvas_term_id
+        expected_job_chain[:global_options][:canvas_term_id] = active_term_1.canvas_term_id
 
         expect(CanvasSync::Jobs::ReportStarter).to receive(:perform_later)
           .with(
@@ -28,7 +28,7 @@ RSpec.describe CanvasSync::Jobs::SyncProvisioningReportJob do
           )
 
         expected_job_chain_2 = Marshal.load(Marshal.dump(job_chain))
-        expected_job_chain_2[:options][:canvas_term_id] = active_term_2.canvas_term_id
+        expected_job_chain_2[:global_options][:canvas_term_id] = active_term_2.canvas_term_id
 
         expect(CanvasSync::Jobs::ReportStarter).to receive(:perform_later)
           .with(

data/spec/canvas_sync/jobs/sync_terms_job_spec.rb

@@ -3,12 +3,13 @@ require 'spec_helper'
 RSpec.describe CanvasSync::Jobs::SyncTermsJob do
   describe '#perform' do
     let(:term_params) { open_canvas_fixture('terms')['enrollment_terms'] }
+    let(:job_chain) { { jobs: [], global_options: {}} }
 
     it 'retrieves all terms from the Canvas API and then invokes the next job' do
-      expect(CanvasSync).to receive(:invoke_next).with([])
+      expect(CanvasSync).to receive(:invoke_next).with(job_chain)
 
       expect {
-        CanvasSync::Jobs::SyncTermsJob.perform_now([], {})
+        CanvasSync::Jobs::SyncTermsJob.perform_now(job_chain, {})
       }.to change { Term.count }.by(term_params.length)
     end
   end

data/spec/canvas_sync/jobs/sync_users_job_spec.rb

@@ -9,7 +9,7 @@ RSpec.describe CanvasSync::Jobs::SyncUsersJob do
 
       expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original
 
-      CanvasSync::Jobs::SyncUsersJob.perform_now([], {})
+      CanvasSync::Jobs::SyncUsersJob.perform_now({ jobs: [], global_options: {}}, {})
     end
   end
 end

data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb

@@ -37,5 +37,12 @@ RSpec.describe CanvasSync::Processors::ProvisioningReportProcessor do
         expect(Course.count).to eq(course_count + 2)
       end
     end
+
+    context 'legacy_support is true' do
+      it 'uses the LegacyImporter' do
+        expect(User).to receive(:create_or_update_from_csv).at_least(:once)
+        subject.process('spec/support/fixtures/reports/users.csv', { models: ['users'], legacy_support: true })
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.0
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-19 00:00:00.000000000 Z
+date: 2017-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -226,14 +226,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.1.0
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.1.0
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: bearcat
   requirement: !ruby/object:Gem::Requirement
@@ -301,6 +301,7 @@ files:
 - lib/canvas_sync/generators/templates/term.rb
 - lib/canvas_sync/generators/templates/user.rb
 - lib/canvas_sync/importers/bulk_importer.rb
+- lib/canvas_sync/importers/legacy_importer.rb
 - lib/canvas_sync/job.rb
 - lib/canvas_sync/jobs/report_checker.rb
 - lib/canvas_sync/jobs/report_processor_job.rb
@@ -345,16 +346,12 @@ files:
 - spec/dummy/config/initializers/wrap_parameters.rb
 - spec/dummy/config/routes.rb
 - spec/dummy/config/secrets.yml
-- spec/dummy/db/development.sqlite3
 - spec/dummy/db/migrate/20170905192509_create_enrollments.rb
 - spec/dummy/db/migrate/20170906193506_create_terms.rb
 - spec/dummy/db/migrate/20170906203438_create_sections.rb
 - spec/dummy/db/migrate/20170914181345_create_courses.rb
 - spec/dummy/db/migrate/20170918221413_create_users.rb
 - spec/dummy/db/schema.rb
-- spec/dummy/db/test.sqlite3
-- spec/dummy/log/development.log
-- spec/dummy/log/test.log
 - spec/factories/course_factory.rb
 - spec/factories/enrollment_factory.rb
 - spec/factories/section_factory.rb
@@ -366,8 +363,6 @@ files:
 - spec/support/fixtures/reports/courses.csv
 - spec/support/fixtures/reports/enrollments.csv
 - spec/support/fixtures/reports/provisioning_csv
-- spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
-- spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv
 - spec/support/fixtures/reports/sections.csv
 - spec/support/fixtures/reports/users.csv
 homepage: https://instructure.com
@@ -429,16 +424,12 @@ test_files:
 - spec/dummy/config/initializers/wrap_parameters.rb
 - spec/dummy/config/routes.rb
 - spec/dummy/config/secrets.yml
-- spec/dummy/db/development.sqlite3
 - spec/dummy/db/migrate/20170905192509_create_enrollments.rb
 - spec/dummy/db/migrate/20170906193506_create_terms.rb
 - spec/dummy/db/migrate/20170906203438_create_sections.rb
 - spec/dummy/db/migrate/20170914181345_create_courses.rb
 - spec/dummy/db/migrate/20170918221413_create_users.rb
 - spec/dummy/db/schema.rb
-- spec/dummy/db/test.sqlite3
-- spec/dummy/log/development.log
-- spec/dummy/log/test.log
 - spec/factories/course_factory.rb
 - spec/factories/enrollment_factory.rb
 - spec/factories/section_factory.rb
@@ -450,7 +441,5 @@ test_files:
 - spec/support/fixtures/reports/courses.csv
 - spec/support/fixtures/reports/enrollments.csv
 - spec/support/fixtures/reports/provisioning_csv
-- spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
-- spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv
 - spec/support/fixtures/reports/sections.csv
 - spec/support/fixtures/reports/users.csv