canvas_sync 0.1.1

data/lib/canvas_sync/generators/templates/create_users.rb

@@ -0,0 +1,18 @@
+<%= autogenerated_migration_warning %>
+
+class CreateUsers < ActiveRecord::Migration[5.1]
+  def change
+    create_table :users do |t|
+      t.bigint :canvas_user_id, null: false
+      t.string :sis_id
+      t.string :email
+      t.string :first_name
+      t.string :last_name
+      t.string :status
+
+      t.timestamps
+    end
+
+    add_index :users, :canvas_user_id, unique: true
+  end
+end

data/lib/canvas_sync/generators/templates/enrollment.rb

@@ -0,0 +1,8 @@
+<%= autogenerated_model_warning %>
+
+class Enrollment < ApplicationRecord
+  validates :canvas_enrollment_id, uniqueness: true, presence: true
+  belongs_to :user, primary_key: :canvas_user_id, foreign_key: :canvas_user_id, optional: true
+  belongs_to :course, primary_key: :canvas_course_id, foreign_key: :canvas_course_id, optional: true
+  belongs_to :section, primary_key: :canvas_section_id, foreign_key: :canvas_section_id, optional: true
+end

data/lib/canvas_sync/generators/templates/section.rb

@@ -0,0 +1,7 @@
+<%= autogenerated_model_warning %>
+
+class Section < ApplicationRecord
+  validates :canvas_section_id, uniqueness: true, presence: true
+  belongs_to :course, primary_key: :canvas_course_id, foreign_key: :canvas_course_id, optional: true
+  has_many :enrollments, primary_key: :canvas_section_id, foreign_key: :canvas_section_id
+end

data/lib/canvas_sync/generators/templates/term.rb

@@ -0,0 +1,28 @@
+<%= autogenerated_model_warning %>
+
+class Term < ApplicationRecord
+  validates :canvas_term_id, uniqueness: true, presence: true
+  has_many :courses, foreign_key: :canvas_term_id, primary_key: :canvas_term_id
+
+  # This is a sample scope created by the CanvasSync gem; feel
+  # free to customize it for your tool's requirements.
+  scope :active, -> {
+    where(workflow_state: 'active')
+      .where("start_at <= ? OR start_at IS NULL", 15.days.from_now)
+      .where("end_at >= ? OR end_at IS NULL", 15.days.ago)
+  }
+
+  def self.create_or_update(term_params)
+    term = Term.find_or_initialize_by(canvas_term_id: term_params['id'])
+
+    term.assign_attributes(name: term_params['name'],
+                           start_at: term_params['start_at'],
+                           end_at: term_params['end_at'],
+                           workflow_state: term_params['workflow_state'],
+                           grading_period_group_id: term_params['grading_period_group_id'],
+                           sis_id: term_params['sis_term_id'])
+
+    term.save! if term.changed?
+    term
+  end
+end

data/lib/canvas_sync/generators/templates/user.rb

@@ -0,0 +1,6 @@
+<%= autogenerated_model_warning %>
+
+class User < ApplicationRecord
+  validates :canvas_user_id, uniqueness: true, presence: true
+  has_many :enrollments, primary_key: :canvas_user_id, foreign_key: :canvas_user_id
+end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -0,0 +1,54 @@
+module CanvasSync
+  module Importers
+    class BulkImporter
+      # The batch import size can be customized by setting
+      # the 'BULK_IMPORTER_BATCH_SIZE' environment variable
+      DEFAULT_BATCH_SIZE = 10_000
+
+      # Does a bulk import of a set of models using the activerecord-import gem.
+      #
+      # @param report_file_path [String] path to the report CSV
+      # @param mapping [Hash] a hash of the values to import. The keys are the CSV column names and
+      #   the values are the database column names. {CanvasSync::Processors::ProvisioningReportProcessor::USERS_CSV_MAPPING Example}
+      # @param klass [Object] e.g., User
+      # @param conflict_target [Symbol] represents the database column that will determine if we need to update
+      #   or insert a given row. e.g.,: canvas_user_id
+      # @yieldparam [Array] row if a block is passed in it will yield the current row from the CSV.
+      #   This can be used if you need to filter or massage the data in any way.
+      def self.import(report_file_path, mapping, klass, conflict_target)
+        csv_column_names = mapping.keys
+        database_column_names = mapping.values
+        rows = []
+
+        CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
+          row = yield(row) if block_given?
+          next if row.nil?
+          rows << csv_column_names.map { |column| row[column] }
+
+          if rows.length >= batch_size
+            perform_import(klass, database_column_names, rows, conflict_target)
+            rows = []
+          end
+        end
+
+        perform_import(klass, database_column_names, rows, conflict_target)
+      end
+
+      private
+
+      def self.perform_import(klass, database_column_names, rows, conflict_target)
+        return if rows.length == 0
+        database_column_names = database_column_names.dup
+        klass.import(database_column_names, rows, validate: false, on_duplicate_key_update: {
+          conflict_target: conflict_target,
+          columns: database_column_names
+        })
+      end
+
+      def self.batch_size
+        batch_size = ENV['BULK_IMPORTER_BATCH_SIZE'].to_i
+        batch_size > 0 ? batch_size : DEFAULT_BATCH_SIZE
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/application_job.rb

@@ -0,0 +1,25 @@
+require "active_job"
+
+module CanvasSync
+  module Jobs
+    # Inherit from this class to build a Job that integrates
+    # well with this gem.
+    class ApplicationJob < ActiveJob::Base
+      # Generates a Bearcat::Client used for Canvas API calls
+      #
+      # @param canvas_base_url [String]
+      # @param canvas_api_token [String]
+      # @return [Bearcat::Client]
+      def canvas_client(canvas_base_url, canvas_api_token)
+        Bearcat::Client.new(token: canvas_api_token, prefix: canvas_base_url)
+      end
+
+      # Returns the amount of time to wait between report status checks
+      #
+      # @return [Integer]
+      def report_checker_wait_time
+        Rails.env.development? ? 1.second : 30.seconds
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/report_checker.rb

@@ -0,0 +1,48 @@
+module CanvasSync
+  module Jobs
+    # ActiveJob class used to check the status of a pending Canvas report.
+    # Re-enqueues itself if the report is still processing on Canvas.
+    # Enqueues the ReportProcessor when the report has completed.
+    class ReportChecker < ApplicationJob
+      # @param canvas_base_url [String]
+      # @param canvas_api_token [String]
+      # @param job_chain [Hash]
+      # @param report_name [Hash] e.g., 'provisioning_csv'
+      # @param report_id [Integer]
+      # @param processor [String] a stringified report processor class name
+      # @param options [Hash] hash of options that will be passed to the job processor
+      # @return [nil]
+      def perform(canvas_base_url, canvas_api_token, job_chain, report_name, report_id, processor, options)
+        report_status = canvas_client(canvas_base_url, canvas_api_token).report_status('self', report_name, report_id)
+
+        case report_status['status'].downcase
+          when 'complete'
+            CanvasSync::Jobs::ReportProcessorJob.perform_later(
+              canvas_base_url,
+              canvas_api_token,
+              job_chain,
+              report_name,
+              report_status['attachment']['url'],
+              processor,
+              options
+            )
+          when 'error', 'deleted'
+            message = "Report failed to process; status was #{report_status} for report_name: #{report_name}, report_id: #{report_id}"
+            Rails.logger.error(message)
+            raise message
+          else
+            CanvasSync::Jobs::ReportChecker.set(wait: report_checker_wait_time)
+              .perform_later(
+                canvas_base_url,
+                canvas_api_token,
+                job_chain,
+                report_name,
+                report_id,
+                processor,
+                options
+              )
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/report_processor_job.rb

@@ -0,0 +1,36 @@
+require 'open-uri'
+
+module CanvasSync
+  module Jobs
+    # ActiveJob class that wraps around a report processor. This job will
+    # download the report, and then pass the file path and options into the
+    # process method on the processor.
+    class ReportProcessorJob < ApplicationJob
+      # @param canvas_base_url [String]
+      # @param canvas_api_token [String]
+      # @param job_chain [Hash]
+      # @param report_name [Hash] e.g., 'provisioning_csv'
+      # @param report_url [String]
+      # @param processor [String] a stringified report processor class name
+      # @param options [Hash] hash of options that will be passed to the job processor
+      # @return [nil]
+      def perform(canvas_base_url, canvas_api_token, job_chain, report_name, report_url, processor, options)
+        download(report_name, report_url) do |file_path|
+          processor.constantize.process(file_path, options)
+        end
+
+        CanvasSync.invoke_next(canvas_base_url, canvas_api_token, job_chain)
+      end
+
+      private
+
+      def download(report_name, report_url)
+        Dir.mktmpdir do |dir|
+          file_path = "#{dir}/#{report_name}"
+          IO.copy_stream(open(report_url), file_path)
+          yield file_path
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/report_starter.rb

@@ -0,0 +1,29 @@
+module CanvasSync
+  module Jobs
+    # Starts a Canvas report and enqueues a ReportChecker
+    class ReportStarter < ApplicationJob
+      # @param canvas_base_url [String]
+      # @param canvas_api_token [String]
+      # @param job_chain [Hash]
+      # @param report_name [Hash] e.g., 'provisioning_csv'
+      # @param report_params [Hash] The Canvas report parameters
+      # @param processor [String] a stringified report processor class name
+      # @param options [Hash] hash of options that will be passed to the job processor
+      # @return [nil]
+      def perform(canvas_base_url, canvas_api_token, job_chain, report_name, report_params, processor, options)
+        client = canvas_client(canvas_base_url, canvas_api_token)
+        report = client.start_report('self', report_name, report_params)
+
+        CanvasSync::Jobs::ReportChecker.set(wait: report_checker_wait_time).perform_later(
+          canvas_base_url,
+          canvas_api_token,
+          job_chain,
+          report_name,
+          report['id'],
+          processor,
+          options
+        )
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -0,0 +1,58 @@
+module CanvasSync
+  module Jobs
+    # ActiveJob class that starts a Canvas provisioning report
+    class SyncProvisioningReportJob < ApplicationJob
+      # @param canvas_base_url [String]
+      # @param canvas_api_token [String]
+      # @param job_chain [Hash]
+      # @param options [Hash] If options contains a :term_scope a seperate provisioning report
+      #   will be started for each term in that scope. :models should be an array of
+      #   models to sync.
+      def perform(canvas_base_url, canvas_api_token, job_chain, options)
+        @canvas_base_url = canvas_base_url
+        @canvas_api_token = canvas_api_token
+        @job_chain = job_chain
+
+        if options[:term_scope]
+          Term.send(options[:term_scope]).find_each do |term|
+            # Deep copy the options so each report gets the correct term id passed into
+            # its options with no side effects
+            duped_options = Marshal.load(Marshal.dump(options))
+            duped_options[:canvas_term_id] = term.canvas_term_id
+            start_report(report_params(options, term.canvas_term_id), duped_options)
+          end
+        else
+          start_report(report_params(options), options)
+        end
+      end
+
+      private
+
+      def start_report(report_params, options)
+        CanvasSync::Jobs::ReportStarter.perform_later(
+          @canvas_base_url,
+          @canvas_api_token,
+          @job_chain,
+          'proservices_provisioning_csv',
+          report_params,
+          CanvasSync::Processors::ProvisioningReportProcessor.to_s,
+          options
+        )
+      end
+
+      def report_params(options, canvas_term_id=nil)
+        params = {
+          "parameters[include_deleted]" => true
+        }
+
+        options[:models].each do |model|
+          params["parameters[#{model}]"] = true
+        end
+
+        params["parameters[enrollment_term_id]"] = canvas_term_id if canvas_term_id
+
+        params
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/sync_terms_job.rb

@@ -0,0 +1,23 @@
+module CanvasSync
+  module Jobs
+    class SyncTermsJob < ApplicationJob
+      # Syncs Terms using the Canvas API
+      #
+      # Terms are pre-synced so that provisioning reports can be scoped to term.
+      #
+      # @param canvas_base_url [String]
+      # @param canvas_api_token [String]
+      # @param job_chain [Hash]
+      # @param options [Hash]
+      def perform(canvas_base_url, canvas_api_token, job_chain, options)
+        client = canvas_client(canvas_base_url, canvas_api_token)
+
+        client.terms('self').all_pages!.each do |term_params|
+          Term.create_or_update(term_params)
+        end
+
+        CanvasSync.invoke_next(canvas_base_url, canvas_api_token, job_chain)
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/sync_users_job.rb

@@ -0,0 +1,32 @@
+module CanvasSync
+  module Jobs
+    class SyncUsersJob < ReportStarter
+      REPORT_PARAMS = {
+        'parameters[users]' => true,
+        'parameters[include_deleted]' => true
+      }
+
+      # Starts a provisioning report for just users.
+      #
+      # Provisioning reports do not scope users by term, so when we are
+      # running provisioning by term we sync users first so we don't duplicate
+      # the work of syncing all users for each term.
+      #
+      # @param canvas_base_url [String]
+      # @param canvas_api_token [String]
+      # @param job_chain [Hash]
+      # @param options [Hash]
+      def perform(canvas_base_url, canvas_api_token, job_chain, options)
+        super(
+          canvas_base_url,
+          canvas_api_token,
+          job_chain,
+          'proservices_provisioning_csv',
+          REPORT_PARAMS,
+          CanvasSync::Processors::ProvisioningReportProcessor.to_s,
+          { models: ['users'] }
+        )
+      end
+    end
+  end
+end

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -0,0 +1,118 @@
+require 'csv'
+require 'activerecord-import'
+require 'zip'
+
+module CanvasSync
+  module Processors
+    class ProvisioningReportProcessor
+      # Used by the {CanvasSync::Importers::BulkImporter bulk importer}. The keys are
+      # CSV columns and the values are the database columns.
+      USERS_CSV_MAPPING = {
+        canvas_user_id: :canvas_user_id,
+        user_id: :sis_id,
+        email: :email,
+        first_name: :first_name,
+        last_name: :last_name,
+        status: :status
+      }
+
+      COURSES_CSV_MAPPING = {
+        canvas_course_id: :canvas_course_id,
+        course_id: :sis_id,
+        short_name: :short_name,
+        long_name: :long_name,
+        canvas_account_id: :canvas_account_id,
+        canvas_term_id: :canvas_term_id,
+        term_id: :term_sis_id,
+        start_date: :start_date,
+        end_date: :end_date
+      }
+
+      ENROLLMENTS_CSV_MAPPING = {
+        canvas_enrollment_id: :canvas_enrollment_id,
+        canvas_course_id: :canvas_course_id,
+        course_id: :course_sis_id,
+        canvas_user_id: :canvas_user_id,
+        user_id: :user_sis_id,
+        role: :role,
+        role_id: :role_id,
+        canvas_section_id: :canvas_section_id,
+        section_id: :section_sis_id,
+        status: :status,
+        base_role_type: :base_role_type
+      }
+
+      SECTIONS_CSV_MAPPING = {
+        canvas_section_id: :canvas_section_id,
+        section_id: :sis_id,
+        canvas_course_id: :canvas_course_id,
+        name: :name,
+        status: :status,
+        start_date: :start_date,
+        end_date: :end_date
+      }
+
+      # Processes a provisioning report using the bulk importer.
+      #
+      # options must contain a models key. If there is only one model
+      # Canvas downloads the single report directly as a CSV. If it's
+      # more than one model Canvas downloads a ZIP file, so we have to
+      # extract that and iterate through it for processing.
+      #
+      # @param report_file_path [String]
+      # @param options [Hash]
+      def self.process(report_file_path, options)
+        if options[:models].length == 1
+          send("process_#{options[:models][0]}", report_file_path)
+        else
+          unzipped_file_path = extract(report_file_path)
+          Dir[unzipped_file_path + "/*.csv"].each do |file_path|
+            model_name = file_path.split("/").last.split(".").first
+            send("process_#{model_name}", file_path)
+          end
+        end
+      end
+
+      private
+
+      def self.extract(file_path)
+        unzipped_file_path = "#{file_path}_unzipped"
+
+        Zip::File.open(file_path) do |zip_file|
+          zip_file.each do |f|
+            f_path = File.join(unzipped_file_path, f.name)
+            FileUtils.mkdir_p(File.dirname(f_path))
+            zip_file.extract(f, f_path) unless File.exist?(f_path)
+          end
+        end
+
+        unzipped_file_path
+      end
+
+      def self.process_users(report_file_path)
+        rows = {}
+
+        # Users can show up more than once in a report if they have multiple logins. This breaks
+        # the bulk import, so the block we pass into the importer filters out users we've already
+        # encountered.
+        CanvasSync::Importers::BulkImporter.import(report_file_path, USERS_CSV_MAPPING, User, :canvas_user_id) do |row|
+          next nil if rows[row[:canvas_user_id]]
+          rows[row[:canvas_user_id]] = true
+          row
+        end
+      end
+
+      def self.process_courses(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(report_file_path, COURSES_CSV_MAPPING, Course, :canvas_course_id)
+      end
+
+      def self.process_enrollments(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(report_file_path, ENROLLMENTS_CSV_MAPPING, Enrollment, :canvas_enrollment_id)
+      end
+
+      def self.process_sections(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(report_file_path, SECTIONS_CSV_MAPPING, Section, :canvas_section_id)
+      end
+    end
+  end
+end