canvas_sync 0.13.0 → 0.16.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c8386afbd924c0148b5c2158d23ea7acd5b0b85349da51312103e1ea9368c54
-  data.tar.gz: 051471b40b5a464b66df8ca7ae482874f2791c55604e28f7fe9ce54cb07f875a
+  metadata.gz: 3c6655d28b3ac244a94cd59da9f76f6878658bff43d1ca071dd7718b82a505ad
+  data.tar.gz: e8ee3801761dd8a84d80ea162ff11e079b219e1d1992788aecd49451c59efaf4
 SHA512:
-  metadata.gz: '00742595c4797ba758ef98ab8b034545eb2193a2cae92dfc585e720c7cd9adef663ba0aadcae2c7085e19497ff74461c6d5553d49114f42dac520c8fe48ca448'
-  data.tar.gz: d8c7eb61e630bb65ea9aa2e86b86c03c2c1f87c151e127314c938f7d1b2438c90dfc5f5ddcc4cb74fe0b525bc4ef9c03dfb140d09fc10045b2f7f56b41524e43
+  metadata.gz: 47a9f7e858cb510f191d1c62f99f112384dc62d3d9d62365369bad5b275d0168d1d5ace94a32b7f76921ee072ae49d3b940e6d56c2aeb51601e9daf52c86b36e
+  data.tar.gz: 84c5b5c9816a2ff8578bb32892739a8737fb6049526f24ef1416d7a5f404fc385f808ca9a7739b1d971baf06f81fe172351015332fe56a67a2e64417158e60d3

data/lib/canvas_sync.rb

@@ -22,7 +22,10 @@ Dir[File.dirname(__FILE__) +  "/canvas_sync/concerns/**/*.rb"].each { |file| req
 module CanvasSync
   SUPPORTED_MODELS = %w[
     users
+    pseudonyms
     courses
+    groups
+    group_memberships
     accounts
     terms
     enrollments
@@ -143,6 +146,10 @@ module CanvasSync
       terms.each do |t|
         return scope.send(t) if scope.respond_to?(t)
       end
+      model = scope.try(:model) || scope
+      if model.try(:column_names)&.include?(:workflow_state)
+        return scope.where.not(workflow_state: %w[deleted])
+      end
       Rails.logger.warn("Could not filter Syncable Scope for model '#{scope.try(:model)&.name || scope.name}'")
       scope
     end
@@ -193,7 +200,6 @@ module CanvasSync
       model_job_map = {
         terms: CanvasSync::Jobs::SyncTermsJob,
         accounts: CanvasSync::Jobs::SyncAccountsJob,
-        users: CanvasSync::Jobs::SyncUsersJob,
         roles: CanvasSync::Jobs::SyncRolesJob,
         admins: CanvasSync::Jobs::SyncAdminsJob,
 
@@ -221,35 +227,41 @@ module CanvasSync
 
       # Accounts, users, roles, and admins are synced before provisioning because they cannot be scoped to term
       try_add_model_job.call('accounts')
-      try_add_model_job.call('users') if term_scope.present?
+
+      # These Models use the provisioning report, but are not term-scoped,
+      # so we sync them before to ensure work is not duplicated
+      if term_scope.present?
+        models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
+        jobs.concat(
+          generate_provisioning_jobs(first_provisioning_models, options)
+        )
+      end
+
       try_add_model_job.call('roles')
       try_add_model_job.call('admins')
-
       pre_provisioning_jobs = jobs
-      jobs = []
 
       ###############################
       # Post provisioning report jobs
       ###############################
 
+      jobs = []
       try_add_model_job.call('assignments')
       try_add_model_job.call('submissions')
       try_add_model_job.call('assignment_groups')
       try_add_model_job.call('context_modules')
       try_add_model_job.call('context_module_items')
-
       post_provisioning_jobs = jobs
 
-      jobs = pre_provisioning_jobs
-      if models.present?
-        provisioning_job = {
-          job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s,
-          options: { term_scope: term_scope, models: models },
-        }
-        provisioning_job[:options].merge!(options[:provisioning]) if options[:provisioning].present?
-        jobs += Array.wrap(provisioning_job)
-      end
-      jobs += post_provisioning_jobs
+      ###############################
+      # Main provisioning job and queueing
+      ###############################
+
+      jobs = [
+        *pre_provisioning_jobs,
+        *generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']),
+        *post_provisioning_jobs,
+      ]
 
       global_options = { legacy_support: legacy_support }
       global_options[:account_id] = account_id if account_id.present?
@@ -258,6 +270,48 @@ module CanvasSync
       JobChain.new(jobs: jobs, global_options: global_options)
     end
 
+    def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
+      dup_models = [ *model_list ]
+      unique_option_models = {}
+
+      filtered_models = only_split ? (only_split & model_list) : model_list
+      filtered_models.each do |m|
+        mopts = options_hash[m.to_sym] || options_hash[default_key]
+        unique_option_models[mopts] ||= []
+        unique_option_models[mopts] << m
+        dup_models.delete(m)
+      end
+
+      if dup_models.present?
+        mopts = options_hash[default_key]
+        unique_option_models[mopts] ||= []
+        unique_option_models[mopts].concat(dup_models)
+      end
+
+      unique_option_models
+    end
+
+    def generate_provisioning_jobs(model_list, options_hash, job_options: {}, only_split: nil, default_key: :provisioning)
+      # Group the model options as best we can.
+      # This is mainly for backwards compatibility, since 'users' was previously it's own job
+      unique_option_models = group_by_job_options(
+        model_list,
+        options_hash,
+        only_split: only_split,
+        default_key: default_key,
+      )
+
+      unique_option_models.map do |mopts, models|
+        opts = { models: models }
+        opts.merge!(job_options)
+        opts.merge!(mopts) if mopts.present?
+        {
+          job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s,
+          options: opts,
+        }
+      end
+    end
+
     # Calls the canvas_sync_client in your app. If you have specified an account
     # ID when starting the job it will pass the account ID to your canvas_sync_client method.
     #

data/lib/canvas_sync/concerns/api_syncable.rb

@@ -146,7 +146,7 @@ module CanvasSync::Concerns
     # Apply a response Hash from the API to this model's attributes and save if changed?
     # @param [Hash] api_params API-format Hash
     # @return [self] self
-    def update_from_api_params(api_params, **kwargs)
+    def update_from_api_params(*args)
       assign_from_api_params(*args)
       save if changed?
     end

data/lib/canvas_sync/concerns/legacy_columns.rb

@@ -15,13 +15,14 @@ module CanvasSync::Concerns
       private
 
       def legacy_column_apply(cls)
-        cid_column = "canvas_#{subclass.name.downcase}_id"
-        column_names = subclass.columns.map(&:name)
+        return if cls.abstract_class
+        cid_column = "canvas_#{cls.name.downcase}_id"
+        column_names = cls.columns.map(&:name)
         return if column_names.include?('canvas_id') && column_names.include?(cid_column)
         if column_names.include?('canvas_id')
-          subclass.alias_attribute(cid_column.to_sym, :canvas_id)
+          cls.alias_attribute(cid_column.to_sym, :canvas_id)
         elsif column_names.include?(cid_column)
-          subclass.alias_attribute(:canvas_id, cid_column.to_sym)
+          cls.alias_attribute(:canvas_id, cid_column.to_sym)
         end
       rescue ActiveRecord::StatementInvalid
       end

data/lib/canvas_sync/generators/templates/migrations/create_group_memberships.rb

@@ -0,0 +1,18 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateGroupMemberships < ActiveRecord::Migration[5.1]
+  def change
+    create_table :group_memberships do |t|
+      t.bigint :canvas_id, null: false
+      t.bigint :canvas_user_id, null: false
+      t.bigint :canvas_group_id, null: false
+      t.string :group_sis_id
+      t.string :user_sis_id
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :group_memberships, :canvas_id, unique: true
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_groups.rb

@@ -0,0 +1,23 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateGroups < ActiveRecord::Migration[5.1]
+  def change
+    create_table :groups do |t|
+      t.bigint :canvas_id, null: false
+      t.string :sis_id
+      t.bigint :canvas_group_category_id
+      t.string :group_category_sis_id
+      t.bigint :canvas_account_id
+      t.bigint :canvas_course_id
+      t.string :name
+      t.string :workflow_state
+      t.bigint :context_id
+      t.string :context_type
+      t.integer :max_membership
+
+      t.timestamps
+    end
+
+    add_index :groups, :canvas_id, unique: true
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_pseudonyms.rb

@@ -0,0 +1,18 @@
+# <%= autogenerated_migration_warning %>
+
+class CreatePseudonyms < ActiveRecord::Migration[5.1]
+  def change
+    create_table :pseudonyms do |t|
+      t.bigint :canvas_id, null: false
+      t.bigint :canvas_user_id
+      t.string :sis_id
+      t.string :unique_id
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :pseudonyms, :canvas_id, unique: true
+    add_index :pseudonyms, :canvas_user_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_submissions.rb

@@ -8,6 +8,7 @@ class CreateSubmissions < ActiveRecord::Migration[5.1]
       t.bigint :canvas_assignment_id
       t.bigint :canvas_user_id
       t.datetime :submitted_at
+      t.datetime :due_at
       t.datetime :graded_at
       t.float :score
       t.float :points_possible

data/lib/canvas_sync/generators/templates/models/account.rb

@@ -12,6 +12,10 @@ class Account < ApplicationRecord
                              primary_key: :canvas_id, foreign_key: :canvas_parent_account_id
   has_many :sub_accounts, class_name: 'Account',
                           primary_key: :canvas_id, foreign_key: :canvas_parent_account_id
+  has_many :groups, primary_key: :canvas_id, foreign_key: :canvas_account_id
+
+  scope :active, -> { where.not(workflow_state: 'deleted') }
+  # scope :should_canvas_sync, -> { active } # Optional - uses .active if not given
 
   api_syncable({
     name: :name,

data/lib/canvas_sync/generators/templates/models/course.rb

@@ -11,6 +11,7 @@ class Course < ApplicationRecord
   has_many :assignments, as: :context, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type
   has_many :submissions, primary_key: :canvas_id, foreign_key: :canvas_course_id
   has_many :assignment_groups, primary_key: :canvas_id, foreign_key: :canvas_course_id
+  has_many :groups, primary_key: :canvas_id, foreign_key: :canvas_course_id
 
   api_syncable({
     sis_id: :sis_course_id,

data/lib/canvas_sync/generators/templates/models/group.rb

@@ -0,0 +1,19 @@
+# <%= autogenerated_model_warning %>
+
+class Group < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
+  belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
+  has_many :group_memberships, primary_key: :canvas_id, foreign_key: :canvas_group_id
+
+  api_syncable({
+    canvas_id: :id,
+    name: :name,
+    canvas_course_id: :course_id,
+    sis_id: :sis_group_id,
+    workflow_state: ->(r){ 'available' },
+  }, -> (api) { api.group(canvas_id) })
+end

data/lib/canvas_sync/generators/templates/models/group_membership.rb

@@ -0,0 +1,17 @@
+# <%= autogenerated_model_warning %>
+
+class GroupMembership < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :group, primary_key: :canvas_id, foreign_key: :canvas_group_id
+  belongs_to :user, primary_key: :canvas_id, foreign_key: :canvas_user_id
+
+  api_syncable({
+    canvas_id: :id,
+    canvas_group_id: :group_id,
+    canvas_user_id: :user_id,
+    workflow_state: :workflow_state,
+  }, -> (api) { api.group_membership(canvas_group_id, canvas_id) })
+end

data/lib/canvas_sync/generators/templates/models/pseudonym.rb

@@ -0,0 +1,8 @@
+# <%= autogenerated_model_warning %>
+
+class Pseudonym < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :user, primary_key: :canvas_id, foreign_key: :canvas_user_id
+end

data/lib/canvas_sync/generators/templates/models/submission.rb

@@ -14,6 +14,7 @@ class Submission < ApplicationRecord
     canvas_user_id: :user_id,
     submitted_at: :submitted_at,
     graded_at: :graded_at,
+    cached_due_date: :due_at,
     score: :score,
     excused: :excused,
     workflow_state: :workflow_state,

data/lib/canvas_sync/generators/templates/models/user.rb

@@ -5,10 +5,12 @@ class User < ApplicationRecord
   include CanvasSync::Concerns::ApiSyncable
 
   validates :canvas_id, uniqueness: true, presence: true
+  has_many :pseudonyms, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :enrollments, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :admins, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :admin_roles, through: :admins, source: :role
   has_many :submissions, primary_key: :canvas_id, foreign_key: :canvas_user_id
+  has_many :group_memberships, primary_key: :canvas_id, foreign_key: :canvas_user_id
 
   api_syncable({
     sis_id: :sis_user_id,

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -49,6 +49,8 @@ module CanvasSync
         }
 
         options[:models].each do |model|
+          # group_membership is the only model param that is singular :(
+          model = 'group_membership' if model == 'group_memberships'
           params[model] = true
         end
 

data/lib/canvas_sync/processors/model_mappings.yml

@@ -29,6 +29,25 @@ users:
       database_column_name: login_id
       type: string
 
+pseudonyms:
+  conflict_target: pseudonym_id
+  report_columns:
+    pseudonym_id:
+      database_column_name: canvas_id
+      type: integer
+    canvas_user_id:
+      database_column_name: canvas_user_id
+      type: integer
+    sis_user_id:
+      database_column_name: sis_id
+      type: string
+    unique_id:
+      database_column_name: unique_id
+      type: string
+    status:
+      database_column_name: workflow_state
+      type: string
+
 accounts:
   conflict_target: canvas_account_id
   report_columns:
@@ -230,6 +249,9 @@ submissions:
     submission_date:
       database_column_name: submitted_at
       type: datetime
+    due_at:
+      database_column_name: due_at
+      type: datetime
     graded_date:
       database_column_name: graded_at
       type: datetime
@@ -326,3 +348,62 @@ context_module_items:
     assignment_id:
       database_column_name: canvas_assignment_id
       type: integer
+
+groups:
+  conflict_target: canvas_group_id
+  report_columns:
+    canvas_group_id:
+      database_column_name: canvas_id
+      type: integer
+    group_id:
+      database_column_name: sis_id
+      type: string
+    canvas_group_category_id:
+      database_column_name: canvas_group_category_id
+      type: integer
+    group_category_id:
+      database_column_name: group_category_sis_id
+      type: integer
+    canvas_account_id:
+      database_column_name: canvas_account_id
+      type: integer
+    canvas_course_id:
+      database_column_name: canvas_course_id
+      type: integer
+    name:
+      database_column_name: name
+      type: string
+    status:
+      database_column_name: workflow_state
+      type: string
+    context_id:
+      database_column_name: context_id
+      type: integer
+    context_type:
+      database_column_name: context_type
+      type: string
+    max_membership:
+      database_column_name: max_membership
+      type: integer
+
+group_memberships:
+  conflict_target: canvas_group_membership_id
+  report_columns:
+    canvas_group_membership_id:
+      database_column_name: canvas_id
+      type: integer
+    canvas_group_id:
+      database_column_name: canvas_group_id
+      type: integer
+    group_id:
+      database_column_name: group_sis_id
+      type: string
+    canvas_user_id:
+      database_column_name: canvas_user_id
+      type: integer
+    user_id:
+      database_column_name: user_sis_id
+      type: string
+    status:
+      database_column_name: workflow_state
+      type: string

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -78,6 +78,15 @@ module CanvasSync
         )
       end
 
+      def bulk_process_pseudonyms(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          mapping[:pseudonyms][:report_columns],
+          Pseudonym,
+          mapping[:pseudonyms][:conflict_target].to_sym,
+        )
+      end
+
       def bulk_process_accounts(report_file_path)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
@@ -122,6 +131,25 @@ module CanvasSync
           mapping[:xlist][:conflict_target].to_sym,
         )
       end
+
+      def bulk_process_groups(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          mapping[:groups][:report_columns],
+          Group,
+          mapping[:groups][:conflict_target].to_sym,
+        )
+      end
+
+      # Note that group membership is singular because we override the model name param in sync_provisioning_report_job
+      def bulk_process_group_membership(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          mapping[:group_memberships][:report_columns],
+          GroupMembership,
+          mapping[:group_memberships][:conflict_target].to_sym,
+        )
+      end
     end
   end
 end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.13.0".freeze
+  VERSION = "0.16.2".freeze
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -34,7 +34,7 @@ RSpec.describe CanvasSync do
         expect(chain.chain_data).to eq({
           jobs: [
             { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: { a: 1 } },
-            { job: CanvasSync::Jobs::SyncUsersJob.to_s, options: { b: 2 } },
+            { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'], b: 2 } },
             { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'], c: 3 } }
           ],
           global_options: { legacy_support: false, d: 4 }
@@ -47,7 +47,7 @@ RSpec.describe CanvasSync do
           expect(chain.chain_data).to eq({
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
-              { job: CanvasSync::Jobs::SyncUsersJob.to_s, options: {} },
+              { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'] } },
               { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
             ],
             global_options: { legacy_support: false }
@@ -247,4 +247,7 @@ RSpec.describe CanvasSync do
     end
   end
 
+  describe ".sync_scope" do
+
+  end
 end

data/spec/canvas_sync/models/accounts_spec.rb

@@ -8,4 +8,7 @@ RSpec.describe Account, type: :model do
     it { should validate_uniqueness_of(:canvas_id) }
   end
 
+  describe 'associations' do
+    it { should have_many(:groups) }
+  end
 end

data/spec/canvas_sync/models/course_spec.rb

@@ -41,5 +41,9 @@ RSpec.describe Course, type: :model do
     it do
       should have_many(:submissions)
     end
+
+    it do
+      should have_many(:groups)
+    end
   end
 end

data/spec/canvas_sync/models/group_membership_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+RSpec.describe GroupMembership, type: :model do
+  let(:subject) {
+    FactoryGirl.create(:group_membership, group: FactoryGirl.create(:group), user: FactoryGirl.create(:user))
+  }
+
+  describe 'validations' do
+    it { should validate_presence_of(:canvas_id) }
+    it { should validate_uniqueness_of(:canvas_id) }
+  end
+
+  describe 'associations' do
+    it do
+      should belong_to(:group)
+        .with_primary_key(:canvas_id)
+        .with_foreign_key(:canvas_group_id)
+    end
+
+    it do
+      should belong_to(:user)
+        .with_primary_key(:canvas_id)
+        .with_foreign_key(:canvas_user_id)
+    end
+  end
+end

data/spec/canvas_sync/models/group_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+RSpec.describe Group, type: :model do
+  let(:subject) { FactoryGirl.create(:group) }
+
+  describe 'validations' do
+    it { should validate_presence_of(:canvas_id) }
+    it { should validate_uniqueness_of(:canvas_id) }
+  end
+
+  describe 'associations' do
+    it { should have_many(:group_memberships) }
+
+    it do
+      should belong_to(:course)
+        .with_primary_key(:canvas_id)
+        .with_foreign_key(:canvas_course_id)
+    end
+
+    it do
+      should belong_to(:account)
+        .with_primary_key(:canvas_id)
+        .with_foreign_key(:canvas_account_id)
+    end
+  end
+end

data/spec/canvas_sync/models/user_spec.rb

@@ -9,6 +9,8 @@ RSpec.describe User, type: :model do
   end
 
   describe 'associations' do
+    it { should have_many(:group_memberships) }
+
     it do
       should have_many(:enrollments)
         .with_primary_key(:canvas_id)

data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb

@@ -42,6 +42,26 @@ RSpec.describe CanvasSync::Processors::ProvisioningReportProcessor do
       expect(cross_listed_section.canvas_nonxlist_course_id).to eq 2
     end
 
+    it 'processes groups' do
+      expect {
+        subject.process('spec/support/fixtures/reports/groups.csv', { models: ['groups'] }, 1)
+      }.to change { Group.count }.by(2)
+      group = Group.find_by_name('Group1')
+      expect(group.workflow_state).to eq 'available'
+      expect(group.canvas_account_id).to eq 1
+      expect(group.canvas_id).to eq 50
+    end
+
+    it 'processes group memberships' do
+      expect {
+        subject.process('spec/support/fixtures/reports/group_memberships.csv', { models: ['group_membership'] }, 1)
+      }.to change { GroupMembership.count }.by(2)
+      group = GroupMembership.find_by_canvas_id(50)
+      expect(group.workflow_state).to eq 'accepted'
+      expect(group.canvas_group_id).to eq 1
+      expect(group.canvas_user_id).to eq 10
+    end
+
     context 'options[:models] is multiple models' do
       it 'extracts the ZIP and processes each model' do
         user_count = User.count

data/spec/dummy/app/models/account.rb

@@ -18,6 +18,7 @@ class Account < ApplicationRecord
                              primary_key: :canvas_id, foreign_key: :canvas_parent_account_id
   has_many :sub_accounts, class_name: 'Account',
                           primary_key: :canvas_id, foreign_key: :canvas_parent_account_id
+  has_many :groups, primary_key: :canvas_id, foreign_key: :canvas_account_id
 
   api_syncable({
     name: :name,

data/spec/dummy/app/models/course.rb

@@ -17,6 +17,7 @@ class Course < ApplicationRecord
   has_many :assignments, as: :context, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type
   has_many :submissions, primary_key: :canvas_id, foreign_key: :canvas_course_id
   has_many :assignment_groups, primary_key: :canvas_id, foreign_key: :canvas_course_id
+  has_many :groups, primary_key: :canvas_id, foreign_key: :canvas_course_id
 
   api_syncable({
     sis_id: :sis_course_id,

data/spec/dummy/app/models/group.rb

@@ -0,0 +1,25 @@
+# #
+# AUTO GENERATED MODEL
+# This model was auto generated by the CanvasSync Gem.
+# You can customize it as needed, but make sure you test
+# any changes you make to the auto generated methods.
+#
+
+
+class Group < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
+  belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
+  has_many :group_memberships, primary_key: :canvas_id, foreign_key: :canvas_group_id
+
+  api_syncable({
+    canvas_id: :id,
+    name: :name,
+    canvas_course_id: :course_id,
+    sis_id: :sis_group_id,
+    workflow_state: ->(r){ 'available' },
+  }, -> (api) { api.group(canvas_id) })
+end

data/spec/dummy/app/models/group_membership.rb

@@ -0,0 +1,23 @@
+# #
+# AUTO GENERATED MODEL
+# This model was auto generated by the CanvasSync Gem.
+# You can customize it as needed, but make sure you test
+# any changes you make to the auto generated methods.
+#
+
+
+class GroupMembership < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :group, primary_key: :canvas_id, foreign_key: :canvas_group_id
+  belongs_to :user, primary_key: :canvas_id, foreign_key: :canvas_user_id
+
+  api_syncable({
+    canvas_id: :id,
+    canvas_group_id: :group_id,
+    canvas_user_id: :user_id,
+    workflow_state: :workflow_state,
+  }, -> (api) { api.group_membership(canvas_group_id, canvas_id) })
+end

data/spec/dummy/app/models/user.rb

@@ -15,6 +15,7 @@ class User < ApplicationRecord
   has_many :admins, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :admin_roles, through: :admins, source: :role
   has_many :submissions, primary_key: :canvas_id, foreign_key: :canvas_user_id
+  has_many :group_memberships, primary_key: :canvas_id, foreign_key: :canvas_user_id
 
   api_syncable({
     sis_id: :sis_user_id,

data/spec/dummy/db/migrate/20190702203627_create_submissions.rb

@@ -14,6 +14,7 @@ class CreateSubmissions < ActiveRecord::Migration[5.1]
       t.bigint :canvas_assignment_id
       t.bigint :canvas_user_id
       t.datetime :submitted_at
+      t.datetime :due_at
       t.datetime :graded_at
       t.float :score
       t.float :points_possible

data/spec/dummy/db/migrate/20200415171620_create_groups.rb

@@ -0,0 +1,29 @@
+# #
+# AUTO GENERATED MIGRATION
+# This migration was auto generated by the CanvasSync Gem.
+# You can add new columns to this table, but removing or
+# re-naming ones created here may break Canvas Syncing.
+#
+
+
+class CreateGroups < ActiveRecord::Migration[5.1]
+  def change
+    create_table :groups do |t|
+      t.bigint :canvas_id, null: false
+      t.string :sis_id
+      t.bigint :canvas_group_category_id
+      t.string :group_category_sis_id
+      t.bigint :canvas_account_id
+      t.bigint :canvas_course_id
+      t.string :name
+      t.string :workflow_state
+      t.bigint :context_id
+      t.string :context_type
+      t.integer :max_membership
+
+      t.timestamps
+    end
+
+    add_index :groups, :canvas_id, unique: true
+  end
+end

data/spec/dummy/db/migrate/20200416214248_create_group_memberships.rb

@@ -0,0 +1,24 @@
+# #
+# AUTO GENERATED MIGRATION
+# This migration was auto generated by the CanvasSync Gem.
+# You can add new columns to this table, but removing or
+# re-naming ones created here may break Canvas Syncing.
+#
+
+
+class CreateGroupMemberships < ActiveRecord::Migration[5.1]
+  def change
+    create_table :group_memberships do |t|
+      t.bigint :canvas_id, null: false
+      t.bigint :canvas_user_id, null: false
+      t.bigint :canvas_group_id, null: false
+      t.string :group_sis_id
+      t.string :user_sis_id
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :group_memberships, :canvas_id, unique: true
+  end
+end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20190927204546) do
+ActiveRecord::Schema.define(version: 2020_04_16_214248) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -162,6 +162,35 @@ ActiveRecord::Schema.define(version: 20190927204546) do
     t.index ["canvas_user_id"], name: "index_enrollments_on_canvas_user_id"
   end
 
+  create_table "group_memberships", force: :cascade do |t|
+    t.bigint "canvas_id", null: false
+    t.bigint "canvas_user_id", null: false
+    t.bigint "canvas_group_id", null: false
+    t.string "group_sis_id"
+    t.string "user_sis_id"
+    t.string "workflow_state"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["canvas_id"], name: "index_group_memberships_on_canvas_id", unique: true
+  end
+
+  create_table "groups", force: :cascade do |t|
+    t.bigint "canvas_id", null: false
+    t.string "sis_id"
+    t.bigint "canvas_group_category_id"
+    t.string "group_category_sis_id"
+    t.bigint "canvas_account_id"
+    t.bigint "canvas_course_id"
+    t.string "name"
+    t.string "workflow_state"
+    t.bigint "context_id"
+    t.string "context_type"
+    t.integer "max_membership"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["canvas_id"], name: "index_groups_on_canvas_id", unique: true
+  end
+
   create_table "roles", force: :cascade do |t|
     t.integer "canvas_id", null: false
     t.string "label"
@@ -196,6 +225,7 @@ ActiveRecord::Schema.define(version: 20190927204546) do
     t.bigint "canvas_assignment_id"
     t.bigint "canvas_user_id"
     t.datetime "submitted_at"
+    t.datetime "due_at"
     t.datetime "graded_at"
     t.float "score"
     t.float "points_possible"

data/spec/factories/group_factory.rb

@@ -0,0 +1,8 @@
+FactoryGirl.define do
+  factory :group do
+    canvas_id { SecureRandom.random_number(100_000_000) }
+    sis_id { SecureRandom.hex }
+    name 'Some User Group'
+    workflow_state 'available'
+  end
+end

data/spec/factories/group_membership_factory.rb

@@ -0,0 +1,6 @@
+FactoryGirl.define do
+  factory :group_membership do
+    canvas_id { SecureRandom.random_number(100_000_000) }
+    workflow_state 'accepted'
+  end
+end

data/spec/support/fixtures/reports/group_memberships.csv

@@ -0,0 +1,3 @@
+canvas_group_id,group_id,canvas_user_id,user_id,status,created_by_sis,canvas_group_membership_id
+1,,10,,accepted,false,50
+2,,11,,accepted,false,51

data/spec/support/fixtures/reports/groups.csv

@@ -0,0 +1,3 @@
+canvas_group_id,group_id,canvas_group_category_id,group_category_id,canvas_account_id,account_id,canvas_course_id,course_id,name,status,created_by_sis,context_id,context_type,max_membership
+50,sis1,10,,1,,,,Group1,available,false,1,Account,
+51,sis2,2,,,,1,coursesis,Group2,available,false,1,Course,1

data/spec/support/fixtures/reports/submissions.csv

@@ -1,3 +1,3 @@
-"canvas user id","sis user id","user name","canvas course id","sis course id","course name","assignment id","assignment name","submission date","graded date","score","points possible","submission id","workflow state","excused"
-1,1,userName,1,1,courseName,1,assignmentName,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,1,graded,true
-1,1,userName,1,1,courseName,2,assignment2Name,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,2,submitted,false
+"canvas user id","sis user id","user name","canvas course id","sis course id","course name","assignment id","assignment name","submission date","graded date","score","points possible","submission id","workflow state","excused","due at"
+1,1,userName,1,1,courseName,1,assignmentName,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,1,graded,true,2017-09-27 18:28:15 UTC
+1,1,userName,1,1,courseName,2,assignment2Name,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,2,submitted,false,2017-09-28 18:28:15 UTC

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.16.2
 platform: ruby
 authors:
 - Nate Collings
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-04 00:00:00.000000000 Z
+date: 2020-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -304,7 +304,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - ncollings@instructure.com
 executables: []
@@ -338,6 +338,9 @@ files:
 - lib/canvas_sync/generators/templates/migrations/create_context_modules.rb
 - lib/canvas_sync/generators/templates/migrations/create_courses.rb
 - lib/canvas_sync/generators/templates/migrations/create_enrollments.rb
+- lib/canvas_sync/generators/templates/migrations/create_group_memberships.rb
+- lib/canvas_sync/generators/templates/migrations/create_groups.rb
+- lib/canvas_sync/generators/templates/migrations/create_pseudonyms.rb
 - lib/canvas_sync/generators/templates/migrations/create_roles.rb
 - lib/canvas_sync/generators/templates/migrations/create_sections.rb
 - lib/canvas_sync/generators/templates/migrations/create_submissions.rb
@@ -351,6 +354,9 @@ files:
 - lib/canvas_sync/generators/templates/models/context_module_item.rb
 - lib/canvas_sync/generators/templates/models/course.rb
 - lib/canvas_sync/generators/templates/models/enrollment.rb
+- lib/canvas_sync/generators/templates/models/group.rb
+- lib/canvas_sync/generators/templates/models/group_membership.rb
+- lib/canvas_sync/generators/templates/models/pseudonym.rb
 - lib/canvas_sync/generators/templates/models/role.rb
 - lib/canvas_sync/generators/templates/models/section.rb
 - lib/canvas_sync/generators/templates/models/submission.rb
@@ -387,7 +393,6 @@ files:
 - lib/canvas_sync/jobs/sync_simple_table_job.rb
 - lib/canvas_sync/jobs/sync_submissions_job.rb
 - lib/canvas_sync/jobs/sync_terms_job.rb
-- lib/canvas_sync/jobs/sync_users_job.rb
 - lib/canvas_sync/processors/assignment_groups_processor.rb
 - lib/canvas_sync/processors/assignments_processor.rb
 - lib/canvas_sync/processors/context_module_items_processor.rb
@@ -416,7 +421,6 @@ files:
 - spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
 - spec/canvas_sync/jobs/sync_submissions_job_spec.rb
 - spec/canvas_sync/jobs/sync_terms_job_spec.rb
-- spec/canvas_sync/jobs/sync_users_job_spec.rb
 - spec/canvas_sync/models/accounts_spec.rb
 - spec/canvas_sync/models/admins_spec.rb
 - spec/canvas_sync/models/assignment_group_spec.rb
@@ -425,6 +429,8 @@ files:
 - spec/canvas_sync/models/context_module_spec.rb
 - spec/canvas_sync/models/course_spec.rb
 - spec/canvas_sync/models/enrollment_spec.rb
+- spec/canvas_sync/models/group_membership_spec.rb
+- spec/canvas_sync/models/group_spec.rb
 - spec/canvas_sync/models/roles_spec.rb
 - spec/canvas_sync/models/section_spec.rb
 - spec/canvas_sync/models/submission_spec.rb
@@ -450,6 +456,8 @@ files:
 - spec/dummy/app/models/context_module_item.rb
 - spec/dummy/app/models/course.rb
 - spec/dummy/app/models/enrollment.rb
+- spec/dummy/app/models/group.rb
+- spec/dummy/app/models/group_membership.rb
 - spec/dummy/app/models/role.rb
 - spec/dummy/app/models/section.rb
 - spec/dummy/app/models/submission.rb
@@ -510,6 +518,8 @@ files:
 - spec/dummy/db/migrate/20190702203632_create_context_module_items.rb
 - spec/dummy/db/migrate/20190927204545_create_roles.rb
 - spec/dummy/db/migrate/20190927204546_create_admins.rb
+- spec/dummy/db/migrate/20200415171620_create_groups.rb
+- spec/dummy/db/migrate/20200416214248_create_group_memberships.rb
 - spec/dummy/db/schema.rb
 - spec/dummy/db/test.sqlite3
 - spec/dummy/log/development.log
@@ -522,6 +532,8 @@ files:
 - spec/factories/context_module_item_factory.rb
 - spec/factories/course_factory.rb
 - spec/factories/enrollment_factory.rb
+- spec/factories/group_factory.rb
+- spec/factories/group_membership_factory.rb
 - spec/factories/role_factory.rb
 - spec/factories/section_factory.rb
 - spec/factories/submission_factory.rb
@@ -540,6 +552,8 @@ files:
 - spec/support/fixtures/reports/context_modules.csv
 - spec/support/fixtures/reports/courses.csv
 - spec/support/fixtures/reports/enrollments.csv
+- spec/support/fixtures/reports/group_memberships.csv
+- spec/support/fixtures/reports/groups.csv
 - spec/support/fixtures/reports/provisioning_csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv
@@ -550,7 +564,7 @@ files:
 homepage: https://instructure.com
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -565,8 +579,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Gem for generating Canvas models and migrations and syncing data from Canvas
 test_files:
@@ -586,7 +600,6 @@ test_files:
 - spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
 - spec/canvas_sync/jobs/sync_submissions_job_spec.rb
 - spec/canvas_sync/jobs/sync_terms_job_spec.rb
-- spec/canvas_sync/jobs/sync_users_job_spec.rb
 - spec/canvas_sync/models/accounts_spec.rb
 - spec/canvas_sync/models/admins_spec.rb
 - spec/canvas_sync/models/assignment_group_spec.rb
@@ -595,6 +608,8 @@ test_files:
 - spec/canvas_sync/models/context_module_spec.rb
 - spec/canvas_sync/models/course_spec.rb
 - spec/canvas_sync/models/enrollment_spec.rb
+- spec/canvas_sync/models/group_membership_spec.rb
+- spec/canvas_sync/models/group_spec.rb
 - spec/canvas_sync/models/roles_spec.rb
 - spec/canvas_sync/models/section_spec.rb
 - spec/canvas_sync/models/submission_spec.rb
@@ -620,6 +635,8 @@ test_files:
 - spec/dummy/app/models/context_module_item.rb
 - spec/dummy/app/models/course.rb
 - spec/dummy/app/models/enrollment.rb
+- spec/dummy/app/models/group.rb
+- spec/dummy/app/models/group_membership.rb
 - spec/dummy/app/models/role.rb
 - spec/dummy/app/models/section.rb
 - spec/dummy/app/models/submission.rb
@@ -680,6 +697,8 @@ test_files:
 - spec/dummy/db/migrate/20190702203632_create_context_module_items.rb
 - spec/dummy/db/migrate/20190927204545_create_roles.rb
 - spec/dummy/db/migrate/20190927204546_create_admins.rb
+- spec/dummy/db/migrate/20200415171620_create_groups.rb
+- spec/dummy/db/migrate/20200416214248_create_group_memberships.rb
 - spec/dummy/db/schema.rb
 - spec/dummy/db/test.sqlite3
 - spec/dummy/log/development.log
@@ -692,6 +711,8 @@ test_files:
 - spec/factories/context_module_item_factory.rb
 - spec/factories/course_factory.rb
 - spec/factories/enrollment_factory.rb
+- spec/factories/group_factory.rb
+- spec/factories/group_membership_factory.rb
 - spec/factories/role_factory.rb
 - spec/factories/section_factory.rb
 - spec/factories/submission_factory.rb
@@ -710,6 +731,8 @@ test_files:
 - spec/support/fixtures/reports/context_modules.csv
 - spec/support/fixtures/reports/courses.csv
 - spec/support/fixtures/reports/enrollments.csv
+- spec/support/fixtures/reports/group_memberships.csv
+- spec/support/fixtures/reports/groups.csv
 - spec/support/fixtures/reports/provisioning_csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv

data/lib/canvas_sync/jobs/sync_users_job.rb

@@ -1,26 +0,0 @@
-module CanvasSync
-  module Jobs
-    class SyncUsersJob < ReportStarter
-      # Starts a provisioning report for just users.
-      #
-      # Provisioning reports do not scope users by term, so when we are
-      # running provisioning by term we sync users first so we don't duplicate
-      # the work of syncing all users for each term.
-      #
-      # @param job_chain [Hash]
-      # @param options [Hash]
-      def perform(job_chain, options)
-        super(
-          job_chain,
-          "proservices_provisioning_csv",
-          merge_report_params(job_chain, options, {
-            users: true,
-            include_deleted: true,
-          }, term_scope: false),
-          CanvasSync::Processors::ProvisioningReportProcessor.to_s,
-          { models: ["users"] },
-        )
-      end
-    end
-  end
-end

data/spec/canvas_sync/jobs/sync_users_job_spec.rb

@@ -1,15 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe CanvasSync::Jobs::SyncUsersJob do
-  describe '#perform' do
-    it 'enqueues a ReportStarter for a provisioning report across all terms with just users' do
-      expect_any_instance_of(Bearcat::Client).to receive(:start_report)
-        .with('self', 'proservices_provisioning_csv', { parameters: { users: true, include_deleted: true } })
-        .and_return({ 'id' => 1 })
-
-      expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original
-
-      CanvasSync::Jobs::SyncUsersJob.perform_now({ jobs: [], global_options: {}}, {})
-    end
-  end
-end