canvas_sync 0.17.32 → 0.17.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac101dc1c7a2d590408677aac25218ac20b762d993413c57ae9f8164b50fb3ad
-  data.tar.gz: c1d4785903974a78e8efc0612e5b36b0c342c6fb8bcfb615616c1e6002cb2186
+  metadata.gz: 43e293e4d64336d70983fbf6eedffb1279252c7d95dfe7a525aa1fa2da5f5463
+  data.tar.gz: b13c557e206b4ea989ff962ad385b0fef0862e67a07626eb8a661faae8c96fd2
 SHA512:
-  metadata.gz: 61f0685f874a9476d8130923b191e7ee1389fcf1cd7374f7b482c11c1e073b30a69796f43b02894cfd37800f7d2240e52d5e48291e8b0f8f8c3698d4f3e84938
-  data.tar.gz: d2a76163cf0a26ba32a96811c1d9e5ac63b6cc7bb23255a27416562e71bd25ee7ae1d57c0d8b21c8043701c9f76eed219469cf48c82883d68d403f938d4f6b0a
+  metadata.gz: 1866a0aafdc8eac4471c5e498055a64690f99982218d06a0ea14ac5ad8a496df1db6add14012d967a8de61efcdced32a1ec14aa7d5dc07c2a0299dc837cd6f0e
+  data.tar.gz: e16462ec03aa1a207871ab2dee9d0075a349aa09aaa7c8014833e9c4f3abe684a62e124fbdc5a76bd5340b913add521ec4304eae0f449b95c624358c4504c8f3

data/lib/canvas_sync/concerns/sync_mapping.rb

@@ -88,7 +88,7 @@ module CanvasSync::Concerns
       end
 
       def unlink_column(key)
-        @map_def.delete(key)
+        @map_def[:report_columns].delete(key)
       end
 
       def link_column(m, type: nil, &blk)

data/lib/canvas_sync/engine.rb

@@ -29,27 +29,46 @@ module CanvasSync
       }
     }
 
+    initializer 'canvas_sync.global_methods' do
+      next if defined?(canvas_sync_client)
+
+      require 'panda_pal'
+
+      def canvas_sync_client(account_id = nil)
+        Bearcat::Client.new(
+          prefix: current_organization.canvas_url,
+          token: current_organization.canvas_api_token,
+          master_rate_limit: (Rails.env.production? && !!defined?(Redis) && ENV['REDIS_URL'].present?),
+        )
+      end
+    rescue LoadError
+    end
+
     initializer :integrate_pandapal do
       require 'panda_pal'
 
-      if PandaPal::Organization.respond_to?(:scheduled_task)
-        if PandaPal::Organization.respond_to?(:define_setting)
-          PandaPal::Organization.define_setting(:canvas_sync, {
-            type: 'Hash',
-            required: false,
-            properties: {
-              job_log_retention: { **RETENTION_TYPE },
-              sync_batch_retention: { **RETENTION_TYPE },
-            }
-          })
-        end
+      Rails.application.reloader.to_prepare do
+        if PandaPal::Organization.respond_to?(:scheduled_task)
+          if PandaPal::Organization.respond_to?(:define_setting)
+            PandaPal::Organization.define_setting(:canvas_sync, {
+              type: 'Hash',
+              required: false,
+              properties: {
+                job_log_retention: { **RETENTION_TYPE },
+                sync_batch_retention: { **RETENTION_TYPE },
+              }
+            })
+          end
 
-        PandaPal::Organization.scheduled_task '0 0 3 * * *', :clean_canvas_sync_logs do
-          job_log_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :job_log_retention) || '3 months', keep_zero: true).seconds.ago
-          JobLog.where('updated_at < ?', job_log_retention).delete_all
+          unless PandaPal::Organization.task_scheduled?(:clean_canvas_sync_logs)
+            PandaPal::Organization.scheduled_task '0 0 3 * * *', :clean_canvas_sync_logs do
+              job_log_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :job_log_retention) || '3 months', keep_zero: true).seconds.ago
+              JobLog.where('updated_at < ?', job_log_retention).delete_all
 
-          sync_batch_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :sync_batch_retention) || '6 months', keep_zero: true).seconds.ago
-          SyncBatch.where('updated_at < ?', sync_batch_retention).delete_all
+              sync_batch_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :sync_batch_retention) || '6 months', keep_zero: true).seconds.ago
+              SyncBatch.where('updated_at < ?', sync_batch_retention).delete_all
+            end
+          end
         end
       end
     rescue LoadError

data/lib/canvas_sync/generators/templates/migrations/create_learning_outcomes.rb

@@ -0,0 +1,30 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateLearningOutcomes < ActiveRecord::Migration[5.1]
+  def change
+    create_table :learning_outcomes do |t|
+      t.bigint :canvas_id, null: false
+      t.integer :canvas_context_id
+      t.string :canvas_context_type
+      t.string :name
+      t.string :friendly_name
+      t.string :workflow_state
+      t.datetime :canvas_created_at
+      t.datetime :canvas_updated_at
+      t.string :migration_id
+      t.string :vendor_guid
+      t.string :low_grade
+      t.string :high_grade
+      t.string :calculation_method
+      t.string :calculation_int
+      t.integer :outcome_import_id
+      t.integer :root_account_ids, array: true, default: []
+      t.text :description
+
+      t.timestamps
+    end
+
+    add_index :learning_outcomes, :canvas_id, unique: true
+    add_index :learning_outcomes, [:canvas_context_id, :canvas_context_type], name: "index_learning_outcomes_on_context"
+  end
+end

data/lib/canvas_sync/generators/templates/models/learning_outcome.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# # <%= autogenerated_migration_warning %>
+
+class LearningOutcome < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  belongs_to :context, polymorphic: true, optional: true, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type
+
+  api_syncable({
+    canvas_id: :id,
+    canvas_context_id: :context_id,
+    canvas_context_type: :context_type,
+    name: :title,
+    friendly_name: :display_name,
+    vendor_guid: :vendor_guid,
+    calculation_method: :calculation_method,
+    calculation_int: :calculation_int,
+    description: :description
+  }, ->(api) { api.get("/api/v1/outcomes/#{canvas_id}") })
+end

data/lib/canvas_sync/job_batches/batch.rb

@@ -453,6 +453,23 @@ module CanvasSync
           end
         end
 
+        def uget(key)
+          Batch.redis do |r|
+            case r.type(key)
+            when 'string'
+              r.get(key)
+            when 'list'
+              r.lrange(key, 0, -1)
+            when 'hash'
+              r.hgetall(key)
+            when 'set'
+              r.smembers(key)
+            when 'zset'
+              r.smembers(key, 0, -1)
+            end
+          end
+        end
+
         def method_missing(method_name, *arguments, &block)
           Batch.redis do |r|
             r.send(method_name, *arguments, &block)

data/lib/canvas_sync/job_batches/callback.rb

@@ -27,7 +27,7 @@ module CanvasSync
 
             if clazz && object = Object.const_get(clazz)
               target = target == :instance ? object.new : object
-              if target.respond_to?(method)
+              if target.respond_to?(method, true)
                 target.send(method, status, opts)
               else
                 Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")

data/lib/canvas_sync/job_batches/pool.rb

@@ -44,7 +44,7 @@ module CanvasSync
           wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
           wrapper.jobs {}
 
-          job_desc = job_desc.with_indifferent_access
+          job_desc = job_desc.symbolize_keys
           job_desc = job_desc.merge!(
             job: job_desc[:job].to_s,
             pool_wrapper_batch: wrapper.bid,
@@ -149,7 +149,7 @@ module CanvasSync
             if current_count < limit
               job_desc = pop_job_from_pool
               if job_desc.present?
-                Batch.new(job_desc['pool_wrapper_batch']).jobs do
+                Batch.new(job_desc[:pool_wrapper_batch]).jobs do
                   ChainBuilder.enqueue_job(job_desc)
                 end
                 jobs_added += 1
@@ -170,7 +170,7 @@ module CanvasSync
       def push_job_to_pool(job_desc)
         jobs_key = "#{redis_key}-jobs"
         # This allows duplicate jobs when a Redis Set is used
-        job_desc['_pool_random_key_'] = SecureRandom.urlsafe_base64(10)
+        job_desc[:_pool_random_key_] = SecureRandom.urlsafe_base64(10)
         job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
         order = self.order
 
@@ -204,7 +204,7 @@ module CanvasSync
 
         return nil unless job_json.present?
 
-        ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
+        ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]&.symbolize_keys
       end
 
       def self.redis(&blk)

data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb

@@ -12,7 +12,7 @@ module CanvasSync::JobBatches::Sidekiq
       end
 
       def drain_zset(key)
-        items, _ = Batch.redis do |r|
+        items, _ = CanvasSync::JobBatches::Batch.redis do |r|
           r.multi do |r|
             r.zrange(key, 0, -1)
             r.zremrangebyrank(key, 0, -1)

data/lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb

@@ -10,6 +10,16 @@
                 <th colspan="2" scope=row><%= t('Batch') %></td>
                 <td><%= @batch.bid %></td>
             </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Parent') %></td>
+                <td>
+                    <% if @batch.parent_bid.present? %>
+                        <a href="<%= root_path %>batches/<%= @batch.parent_bid %>"><%= @batch.parent_bid %></a>
+                    <% else %>
+                        ROOT
+                    <% end %>
+                </td>
+            </tr>
             <tr>
                 <th colspan="2" scope=row><%= t('Started') %></td>
                 <td><%= safe_relative_time(@batch.created_at.to_f) %></td>

data/lib/canvas_sync/processors/model_mappings.yml

@@ -509,3 +509,58 @@ content_migrations:
     canvas_root_account_id:
       database_column_name: canvas_root_account_id
       type: integer
+
+learning_outcomes:
+  conflict_target: learning_outcome_id
+  report_columns:
+    learning_outcome_id:
+      database_column_name: canvas_id
+      type: integer
+    context_id:
+      database_column_name: canvas_context_id
+      type: integer
+    context_type:
+      database_column_name: canvas_context_type
+      type: string
+    name:
+      database_column_name: name
+      type: string
+    friendly_name:
+      database_column_name: friendly_name
+      type: string
+    workflow_state:
+      database_column_name: workflow_state
+      type: string
+    created_at:
+      database_column_name: canvas_created_at
+      type: datetime
+    updated_at:
+      database_column_name: canvas_updated_at
+      type: datetime
+    migration_id:
+      database_column_name: migration_id
+      type: string
+    vendor_guid:
+      database_column_name: vendor_guid
+      type: string
+    low_grade:
+      database_column_name: low_grade
+      type: string
+    high_grade:
+      database_column_name: high_grade
+      type: string
+    calculation_method:
+      database_column_name: calculation_method
+      type: string
+    calculation_int:
+      database_column_name: calculation_int
+      type: integer
+    outcome_import_id:
+      database_column_name: outcome_import_id
+      type: integer
+    root_account_ids:
+      database_column_name: root_account_ids
+      type: integer
+    description:
+      database_column_name: description
+      type: string

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -116,6 +116,13 @@ module CanvasSync
       def bulk_process_group_membership(report_file_path)
         do_bulk_import(report_file_path, GroupMembership, options: @options)
       end
+
+      def bulk_process_learning_outcomes(report_file_path)
+        do_bulk_import(report_file_path, LearningOutcome, options: @options) do |row|
+          row[:root_account_ids] = JSON.parse row[:root_account_ids]
+          row
+        end
+      end
     end
   end
 end

data/lib/canvas_sync/processors/report_processor.rb

@@ -12,14 +12,15 @@ module CanvasSync
         model.try(:get_sync_mapping, key) || mapping[key || CanvasSync::Concerns::SyncMapping::Mapping.normalize_model_name(model)]
       end
 
-      def do_bulk_import(report_file_path, model, options: {}, mapping_key: nil)
+      def do_bulk_import(report_file_path, model, options: {}, mapping_key: nil, &blk)
         m = mapping_for(model, mapping_key)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           m[:report_columns],
           model,
           m[:conflict_target],
-          import_args: options
+          import_args: options,
+          &blk
         )
       end
     end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.32".freeze
+  VERSION = "0.17.35".freeze
 end

data/lib/canvas_sync.rb

@@ -45,6 +45,7 @@ module CanvasSync
     grading_periods
     grading_period_groups
     content_migrations
+    learning_outcomes
   ].freeze
 
   SUPPORTED_TERM_SCOPE_MODELS = %w[

data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb

@@ -104,6 +104,12 @@ RSpec.describe CanvasSync::Processors::ProvisioningReportProcessor do
       expect(obj.workflow_state).to eq 'active'
     end
 
+    it 'processes learning_outcomes' do
+      expect {
+        subject.process('spec/support/fixtures/reports/learning_outcomes.csv', { models: ['learning_outcomes'] }, 1)
+      }.to change { LearningOutcome.count }.by(2)
+    end
+
     it 'model with composite key behaves as expected' do
       expect {
         subject.process('spec/support/fixtures/reports/user_observers.csv', { models: ['user_observers'] }, 1)

data/spec/dummy/app/models/learning_outcome.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# # #
+# AUTO GENERATED MIGRATION
+# This migration was auto generated by the CanvasSync Gem.
+# You can add new columns to this table, but removing or
+# re-naming ones created here may break Canvas Syncing.
+#
+
+
+class LearningOutcome < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  belongs_to :context, polymorphic: true, optional: true, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type
+
+  api_syncable({
+    canvas_id: :id,
+    canvas_context_id: :context_id,
+    canvas_context_type: :context_type,
+    name: :title,
+    friendly_name: :display_name,
+    vendor_guid: :vendor_guid,
+    calculation_method: :calculation_method,
+    calculation_int: :calculation_int,
+    description: :description
+  }, ->(api) { api.get("/api/v1/outcomes/#{canvas_id}") })
+end

data/spec/dummy/db/migrate/20220712210559_create_learning_outcomes.rb

@@ -0,0 +1,36 @@
+# #
+# AUTO GENERATED MIGRATION
+# This migration was auto generated by the CanvasSync Gem.
+# You can add new columns to this table, but removing or
+# re-naming ones created here may break Canvas Syncing.
+#
+
+
+class CreateLearningOutcomes < ActiveRecord::Migration[5.1]
+  def change
+    create_table :learning_outcomes do |t|
+      t.bigint :canvas_id, null: false
+      t.integer :canvas_context_id
+      t.string :canvas_context_type
+      t.string :name
+      t.string :friendly_name
+      t.string :workflow_state
+      t.datetime :canvas_created_at
+      t.datetime :canvas_updated_at
+      t.string :migration_id
+      t.string :vendor_guid
+      t.string :low_grade
+      t.string :high_grade
+      t.string :calculation_method
+      t.string :calculation_int
+      t.integer :outcome_import_id
+      t.integer :root_account_ids, array: true, default: []
+      t.text :description
+
+      t.timestamps
+    end
+
+    add_index :learning_outcomes, :canvas_id, unique: true
+    add_index :learning_outcomes, [:canvas_context_id, :canvas_context_type], name: "index_learning_outcomes_on_context"
+  end
+end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2022_03_08_072643) do
+ActiveRecord::Schema.define(version: 2022_07_12_210559) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -246,6 +246,30 @@ ActiveRecord::Schema.define(version: 2022_03_08_072643) do
     t.index ["canvas_id"], name: "index_groups_on_canvas_id", unique: true
   end
 
+  create_table "learning_outcomes", force: :cascade do |t|
+    t.bigint "canvas_id", null: false
+    t.integer "canvas_context_id"
+    t.string "canvas_context_type"
+    t.string "name"
+    t.string "friendly_name"
+    t.string "workflow_state"
+    t.datetime "canvas_created_at"
+    t.datetime "canvas_updated_at"
+    t.string "migration_id"
+    t.string "vendor_guid"
+    t.string "low_grade"
+    t.string "high_grade"
+    t.string "calculation_method"
+    t.string "calculation_int"
+    t.integer "outcome_import_id"
+    t.integer "root_account_ids", default: [], array: true
+    t.text "description"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["canvas_context_id", "canvas_context_type"], name: "index_learning_outcomes_on_context"
+    t.index ["canvas_id"], name: "index_learning_outcomes_on_canvas_id", unique: true
+  end
+
   create_table "pseudonyms", force: :cascade do |t|
     t.bigint "canvas_id", null: false
     t.bigint "canvas_user_id"

data/spec/support/fixtures/reports/learning_outcomes.csv

@@ -0,0 +1,3 @@
+learning_outcome_id,context_id,context_type,name,friendly_name,workflow_state,created_at,updated_at,migration_id,vendor_guid,low_grade,high_grade,calculation_method,calculation_int,outcome_import_id,root_account_ids,description
+1,1,Course,Course Outcome,"",active,2021-08-12 21:11:40 UTC,2021-08-13 01:05:40 UTC,,,,,decaying_average,65,,[1],<p>description</p>
+2,1,Account,Account Outcome,Friendly Name,active,2021-08-12 21:11:40 UTC,2021-08-13 01:05:40 UTC,,,,,decaying_average,65,,[1],<p>description</p>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.17.32
+  version: 0.17.35
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-26 00:00:00.000000000 Z
+date: 2022-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -390,6 +390,7 @@ files:
 - lib/canvas_sync/generators/templates/migrations/create_grading_periods.rb
 - lib/canvas_sync/generators/templates/migrations/create_group_memberships.rb
 - lib/canvas_sync/generators/templates/migrations/create_groups.rb
+- lib/canvas_sync/generators/templates/migrations/create_learning_outcomes.rb
 - lib/canvas_sync/generators/templates/migrations/create_pseudonyms.rb
 - lib/canvas_sync/generators/templates/migrations/create_roles.rb
 - lib/canvas_sync/generators/templates/migrations/create_sections.rb
@@ -410,6 +411,7 @@ files:
 - lib/canvas_sync/generators/templates/models/grading_period_group.rb
 - lib/canvas_sync/generators/templates/models/group.rb
 - lib/canvas_sync/generators/templates/models/group_membership.rb
+- lib/canvas_sync/generators/templates/models/learning_outcome.rb
 - lib/canvas_sync/generators/templates/models/pseudonym.rb
 - lib/canvas_sync/generators/templates/models/role.rb
 - lib/canvas_sync/generators/templates/models/section.rb
@@ -551,6 +553,7 @@ files:
 - spec/dummy/app/models/grading_period_group.rb
 - spec/dummy/app/models/group.rb
 - spec/dummy/app/models/group_membership.rb
+- spec/dummy/app/models/learning_outcome.rb
 - spec/dummy/app/models/pseudonym.rb
 - spec/dummy/app/models/role.rb
 - spec/dummy/app/models/section.rb
@@ -620,6 +623,7 @@ files:
 - spec/dummy/db/migrate/20210907233330_create_grading_periods.rb
 - spec/dummy/db/migrate/20211001184920_create_grading_period_groups.rb
 - spec/dummy/db/migrate/20220308072643_create_content_migrations.rb
+- spec/dummy/db/migrate/20220712210559_create_learning_outcomes.rb
 - spec/dummy/db/schema.rb
 - spec/dummy/log/development.log
 - spec/dummy/log/test.log
@@ -671,6 +675,7 @@ files:
 - spec/support/fixtures/reports/grading_periods.csv
 - spec/support/fixtures/reports/group_memberships.csv
 - spec/support/fixtures/reports/groups.csv
+- spec/support/fixtures/reports/learning_outcomes.csv
 - spec/support/fixtures/reports/provisioning_csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv
@@ -697,7 +702,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Gem for generating Canvas models and migrations and syncing data from Canvas
@@ -759,6 +764,7 @@ test_files:
 - spec/dummy/app/models/grading_period_group.rb
 - spec/dummy/app/models/group.rb
 - spec/dummy/app/models/group_membership.rb
+- spec/dummy/app/models/learning_outcome.rb
 - spec/dummy/app/models/pseudonym.rb
 - spec/dummy/app/models/role.rb
 - spec/dummy/app/models/section.rb
@@ -828,6 +834,7 @@ test_files:
 - spec/dummy/db/migrate/20210907233330_create_grading_periods.rb
 - spec/dummy/db/migrate/20211001184920_create_grading_period_groups.rb
 - spec/dummy/db/migrate/20220308072643_create_content_migrations.rb
+- spec/dummy/db/migrate/20220712210559_create_learning_outcomes.rb
 - spec/dummy/db/schema.rb
 - spec/dummy/log/development.log
 - spec/dummy/log/test.log
@@ -879,6 +886,7 @@ test_files:
 - spec/support/fixtures/reports/grading_periods.csv
 - spec/support/fixtures/reports/group_memberships.csv
 - spec/support/fixtures/reports/groups.csv
+- spec/support/fixtures/reports/learning_outcomes.csv
 - spec/support/fixtures/reports/provisioning_csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv