canvas_sync 0.17.17.beta1 → 0.17.23.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63d76061493c5aa510709d95f5d1e9828f47e675d0571c9a2afc76e2a2b92807
-  data.tar.gz: 320ec24b43c5004c137287a11989dd6982436509f76c8745331ecb1ab6ad4702
+  metadata.gz: 6fe925c97d18da5736681b55e51a1702d727ef2900097c9058de6f247748af9f
+  data.tar.gz: 27c0bd4eb8d57efc705cbe3affdfa6ce4af9580b1b2374e11a8e7e84ddf2b8bd
 SHA512:
-  metadata.gz: dd07a46eb04eeb95a069d516af6a361ca8051d1dce5254a41560ff072472e5090db1c049495b9f2a57a48cba456f127bd7e1df2e8510b205fc053e2398aceb3e
-  data.tar.gz: 62767cc36c1cf7f6850072a9b7ea502e805da4218a4ae63f152f6453d1f02fbf144cca7f791f3ce848eb29c65af57832529f4b6a444e12342cd3aea4c4e77cf2
+  metadata.gz: 82f682b5e2b713ba2adc45483f215c1de8103dc16bdd3872a1656649527053a8819de35b50d1f4bc8e9961bedbc647fd7a03dbebbe4f2ad0fb3235d6754788be
+  data.tar.gz: 674159253800095ba66e15d9d14f58fd41d46f7fa70a33a8303160488544d721f7e8c83700c8a5e4978436e01d34e070b7bc22ff7433219e68fe00d4620dc38f

data/README.md

@@ -190,7 +190,30 @@ Overrides are useful for two scenarios:
 - You have an existing application where the column names do not match up with what CanvasSync expects
 - You want to sync some other column in the report that CanvasSync is not configured to sync
 
-In order to create an override, place a file called `canvas_sync_provisioning_mapping.yml` in your Rails `config` directory. Define the tables and columns you want to override using the following format:
+Mappings can be modified by editing the Model class like such:
+```ruby
+class User < ApplicationRecord
+  include CanvasSync::Record
+
+  sync_mapping(reset: false) do # `reset: false` is the default
+    # The mapping can be totally cleared with `reset: true` in the `sync_mapping` call, or like such:
+    reset_links
+
+    # Add a new column:
+    link_column :column_in_report => :column_in_database, type: :datetime
+
+    # If the column name on the report and in the DB are the same, a shorthand can be used:
+    link_column :omit_from_final_grade, type: :datetime
+
+    # If the defaults define a column you don't want synced, you can remove it from the mapping:
+    unlink_column :column_in_database
+  end
+
+  # ...
+end
+```
+
+You can also create a file called `canvas_sync_provisioning_mapping.yml` in your Rails `config` directory. However, this approach requires you to re-specify the complete table in order to modify a table. Define the tables and columns you want to override using the following format:
 
 ```ruby
 users:
@@ -383,6 +406,27 @@ Available config options (if you add more, please update this!):
 
 * `config.classes_to_only_log_errors_on` - use this if you are utilizing the `CanvasSync::JobLog` table, but want certain classes to only persist in the `job_logs` table if an error is encountered. This is useful if you've got a very frequently used job that's filling up your database, and only really care about tracking failures.
 
+## Global Options
+You can pass in global_options to a job chain.  Global options are added to the batch_context and referenced by
+various internal processes.
+
+Pass global options into a job chain, using the options param nested in a :global key.
+options: { global: {...} }
+
+report_timeout (integer):             Number of days until a Canvas report should timeout.  Default is 1.
+report_compilation_timeout (integer): Number of days until a Canvas report should timeout.  Default is 1 hour.
+                                      You can likely pass a float to achieve sub-day timeouts, but not tested.
+report_max_tries (integer):           The number of times to attempt a report before giving up.  A report is considered failed
+                                      if it has an 'error' status in Canvas or is deleted.
+
+This is an example job chain with global options:
+job_chain = CanvasSync.default_provisioning_report_chain(
+  MODELS_TO_SYNC,
+  term_scope: :active,
+  full_sync_every: 'sunday',
+  options: { global: { report_timeout: 2 } }
+)
+
 ## Handling Job errors
 
 If you need custom handling for when a CanvasSync Job fails, you can add an `:on_failure` option to you Job Chain's `:global_options`.

data/lib/canvas_sync/concerns/sync_mapping.rb

@@ -0,0 +1,112 @@
+module CanvasSync::Concerns
+  module SyncMapping
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def sync_mapping(key = nil, reset: false, &blk)
+        key ||= Mapping.normalize_model_name(self)
+        key = key.to_s
+        existing_map = get_sync_mapping(key)
+        mapper = Mapping.new(existing_map&.deep_dup || {}.with_indifferent_access)
+        mapper.reset_links if reset
+        mapper.instance_exec(&blk)
+        @sync_mappings[key] = mapper.map_def.freeze
+      end
+
+      def get_sync_mapping(key = nil)
+        key ||= Mapping.normalize_model_name(self)
+        key = key.to_s
+        @sync_mappings ||= {}
+        @sync_mappings[key] || superclass.try(:get_sync_mapping, key) || Mapping.default_for(key)
+      end
+    end
+
+    class Mapping
+      attr_reader :map_def
+
+      def initialize(map_def = {}, model: nil)
+        @map_def = map_def
+        @model = model
+      end
+
+      def self.normalize_model_name(model)
+        model = model.name unless model.is_a?(String)
+        model.pluralize.underscore
+      end
+
+      def self.default_for(key)
+        default_mappings[key]
+      end
+
+      def self.default_mappings
+        @mappings ||= begin
+          maps = {}
+          default_v1_mappings.each do |mname, legacy|
+            m = maps[mname] = {}
+
+            m[:conflict_target] = Array(legacy[:conflict_target]).map(&:to_sym).map do |lct|
+              legacy[:report_columns][lct][:database_column_name]
+            end
+
+            m[:report_columns] = {}
+            legacy[:report_columns].each do |rcol, opts|
+              m[:report_columns][opts[:database_column_name]] = opts.except(:database_column_name).merge!(
+                report_column: rcol,
+              ).freeze
+            end
+          end
+          maps.with_indifferent_access.freeze
+        end
+      end
+
+      def self.default_v1_mappings
+        @legacy_mappings ||= begin
+          mapping = YAML.load_file(File.join(__dir__, '../processors', "model_mappings.yml")).deep_symbolize_keys!
+          override_filepath = Rails.root.join("config/canvas_sync_provisioning_mapping.yml")
+
+          if File.file?(override_filepath)
+            override = YAML.load_file(override_filepath).deep_symbolize_keys!
+            mapping = mapping.merge(override)
+          end
+
+          mapping.freeze
+        end
+      end
+
+      def conflict_target(*columns)
+        if columns.count == 0
+          @map_def[:conflict_target]
+        else
+          @map_def[:conflict_target] = columns.flatten.compact
+        end
+      end
+
+      def reset_links
+        @map_def = {}
+      end
+
+      def unlink_column(key)
+        @map_def.delete(key)
+      end
+
+      def link_column(m, type: nil, &blk)
+        if m.is_a?(Hash)
+          raise "Hash should have exactly 1 entry" if m && m.count != 1
+          @map_def[:report_columns][m.values[0]] = {
+            report_column: m.keys[0],
+            type: type,
+            transform: blk,
+          }
+        elsif m.is_a?(Symbol)
+          @map_def[:report_columns][m] = {
+            report_column: m,
+            type: type,
+            transform: blk,
+          }
+        else
+          raise "Cannot handle argument of type #{m.class}"
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/generators/install_generator.rb

@@ -48,6 +48,7 @@ module CanvasSync
       models.each do |model|
         migration_template "migrations/create_#{model}.rb", "db/migrate/create_#{model}.rb"
         template "models/#{model.singularize}.rb", "app/models/#{model.singularize}.rb"
+      rescue
       end
     end
   end

data/lib/canvas_sync/generators/templates/migrations/create_grading_period_groups.rb

@@ -0,0 +1,22 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
+  def change
+    create_table :grading_period_groups do |t|
+      t.bigint :canvas_id, null: false
+      t.bigint :canvas_course_id
+      t.bigint :canvas_account_id
+      t.string :title
+      t.boolean :weighted
+      t.boolean :display_totals_for_all_grading_periods
+
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :grading_period_groups, :canvas_id, unique: true
+    add_index :grading_period_groups, :canvas_course_id
+    add_index :grading_period_groups, :canvas_account_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_grading_periods.rb

@@ -0,0 +1,22 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateGradingPeriods < ActiveRecord::Migration[5.1]
+  def change
+    create_table :grading_periods do |t|
+      t.bigint :canvas_id, null: false
+      t.string :title
+      t.float :weight
+      t.datetime :start_date
+      t.datetime :end_date
+      t.datetime :close_date
+      t.bigint :canvas_grading_period_group_id
+
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :grading_periods, :canvas_id, unique: true
+    add_index :grading_periods, :canvas_grading_period_group_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_user_observers.rb

@@ -0,0 +1,17 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateUserObservers < ActiveRecord::Migration[5.1]
+  def change
+    create_table :user_observers do |t|
+      t.bigint :observing_user_id
+      t.bigint :observed_user_id
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :user_observers, [:observed_user_id, :observing_user_id], unique: true
+    add_index :user_observers, :observing_user_id
+    add_index :user_observers, :observed_user_id
+  end
+end

data/lib/canvas_sync/generators/templates/models/grading_period.rb

@@ -0,0 +1,8 @@
+# <%= autogenerated_model_warning %>
+
+class GradingPeriod < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :canvas_grading_period_group_id, optional: true
+end

data/lib/canvas_sync/generators/templates/models/grading_period_group.rb

@@ -0,0 +1,9 @@
+# <%= autogenerated_model_warning %>
+
+class GradingPeriodGroup < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
+  belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
+end

data/lib/canvas_sync/generators/templates/models/user_observer.rb

@@ -0,0 +1,11 @@
+# <%= autogenerated_model_warning %>
+
+class UserObserver < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+
+  belongs_to :observing_user, primary_key: :canvas_id, foreign_key: :observing_user_id, class_name: 'User', optional: true
+  belongs_to :observed_user, primary_key: :canvas_id, foreign_key: :observed_user_id, class_name: 'User', optional: true
+end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -24,30 +24,41 @@ module CanvasSync
       end
 
       def self.perform_in_batches(report_file_path, mapping, klass, conflict_target, import_args: {})
-        csv_column_names = mapping.keys
-        database_column_names = mapping.values.map { |value| value[:database_column_name] }
-        database_conflict_column_name = conflict_target ? mapping[conflict_target][:database_column_name] : nil
+        csv_column_names = mapping.values.map { |value| value[:report_column].to_s }
+        database_column_names = mapping.keys
+
+        conflict_target = Array(conflict_target).map(&:to_s)
+        conflict_target_indices = conflict_target.map{|ct| database_column_names.index(ct) }
 
         row_ids = {}
         batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
           row_ids = {}
-          perform_import(klass, database_column_names, batch, database_conflict_column_name, import_args)
+          perform_import(klass, database_column_names, batch, conflict_target, import_args)
         end
 
         row_buffer_out = ->(row) {
-          if conflict_target
-            next if row_ids[row[conflict_target]]
-            row_ids[row[conflict_target]] = true
+          formatted_row = mapping.map do |db_col, col_def|
+            value = nil
+            value = row[col_def[:report_column]] if col_def[:report_column]
+
+            if col_def[:type]
+              if col_def[:type].to_sym == :datetime
+                # TODO: add some timezone config to the mapping.
+                # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
+                value = DateTime.parse(value).utc rescue nil # rubocop:disable Style/RescueModifier
+              end
+            end
+
+            value = col_def[:transform].call(value, row) if col_def[:transform]
+
+            value
           end
 
-          formatted_row = csv_column_names.map do |column|
-            if mapping[column][:type].to_sym == :datetime
-              # TODO: add some timezone config to the mapping.
-              # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
-              DateTime.parse(row[column]).utc rescue nil # rubocop:disable Style/RescueModifier
-            else
-              row[column]
-            end
+          if conflict_target.present?
+            key = conflict_target_indices.map{|ct| formatted_row[ct] }
+            next if row_ids[key]
+
+            row_ids[key] = true
           end
 
           batcher << formatted_row
@@ -79,7 +90,7 @@ module CanvasSync
           condition: condition_sql(klass, columns, import_args[:sync_start_time]),
           columns: columns
         }
-        update_conditions[:conflict_target] = conflict_target if conflict_target
+        update_conditions[:conflict_target] = conflict_target if conflict_target.present?
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)

data/lib/canvas_sync/job_batches/batch.rb

@@ -28,6 +28,7 @@ module CanvasSync
 
       BID_EXPIRE_TTL = 2_592_000
       SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
+      BID_HIERARCHY = RedisScript.new(Pathname.new(__FILE__) + "../hier_batch_ids.lua")
 
       attr_reader :bid
 
@@ -423,6 +424,14 @@ module CanvasSync
         def push_callbacks(args, queue)
           Batch::Callback::worker_class.enqueue_all(args, queue)
         end
+
+        def bid_hierarchy(bid, depth: 4, per_depth: 5, slice: nil)
+          args = [bid, depth, per_depth]
+          args << slice if slice
+          redis do |r|
+            BID_HIERARCHY.call(r, [], args)
+          end
+        end
       end
     end
 

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -40,7 +40,7 @@ module CanvasSync
       def insert_at(position, new_jobs)
         chain = self.class.get_chain_parameter(base_job)
         new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
-        chain.insert(-1, *new_jobs)
+        chain.insert(position, *new_jobs)
       end
 
       def insert(new_jobs, **kwargs)
@@ -172,6 +172,14 @@ module CanvasSync
           mapper[key] ||= []
         end
 
+        # TODO: Add a Chain progress web View
+        # Augment Batch tree-view with Chain data
+        #   > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
+        #   > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
+        #     > Chain-jobs will be supplied chain_id and chain_step_id metadata
+        #     > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
+        #     > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
+        #   > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
         def enqueue_job(job_def)
           job_class = job_def[:job].constantize
           job_options = job_def[:parameters] || []

data/lib/canvas_sync/job_batches/hier_batch_ids.lua

@@ -0,0 +1,25 @@
+
+local function add_bids(root, depth)
+    local result_data = {}
+
+    if depth > 0 then
+        local sbids
+        if depth == tonumber(ARGV[2]) and ARGV[4] then
+            local min, max = ARGV[4]:match('(%d+):(%d+)')
+            sbids = redis.call('ZRANGE', 'BID-' .. root .. '-bids', min, max)
+        else
+            sbids = redis.call('ZRANGE', 'BID-' .. root .. '-bids', 0, tonumber(ARGV[3]) - 1)
+        end
+
+        local sub_data = {}
+        for _,v in ipairs(sbids) do
+            table.insert(sub_data, add_bids(v, depth - 1))
+        end
+
+        return { root, sub_data }
+    end
+
+    return { root, result_data}
+end
+
+return add_bids(ARGV[1], tonumber(ARGV[2]))

data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/css/styles.less

@@ -0,0 +1,178 @@
+
+@color-green: #25c766;
+@color-red: #c7254e;
+@color-yellow: #c4c725;
+
+.code-wrap.batch-context .args-extended {
+    white-space: pre;
+
+    .key {
+        white-space: pre-wrap;
+        margin-left: 2em;
+        text-indent: -2em;
+        display: inline-block;
+    }
+
+    .own {
+        color: @color-green;
+    }
+}
+
+
+.batch-tree {
+    .status-block {
+        .tree-stat {
+            margin: 0 4px;
+    
+            &.pending {
+                color: @color-yellow;
+            }
+            &.failed {
+                color: @color-red;
+            }
+            &.success {
+                color: @color-green;
+            }
+            &.total {
+    
+            }
+        }
+    }
+
+    .text-inactive {
+        color: darken(#fff, 35%);
+        font-size: 80%;
+    }
+
+    .tree-header {
+        position: relative;
+
+        .status-block {
+            position: absolute;
+            bottom: 0;
+            width: 100%;
+
+            margin-right: 8px;
+            font-size: 90%;
+            text-align: right;
+
+            .tree-stat {
+                font-style: italic;
+            }
+        }
+    }
+
+    .tree-entry {
+        > .header {
+            display: flex;
+            align-items: center;
+
+            .header-inner {
+                padding: 4px 0;
+                border-bottom: 1px dashed white;
+                display: flex;
+                align-items: center;
+                flex: 1;
+            }
+
+            &:hover {
+                background-color: rgba(0,0,0,0.20);
+                border-radius: 3px;
+            }
+
+            .row-toggle {
+                width: 16px;
+                height: 16px;
+                text-align: center;
+                align-self: center;
+                border-radius: 50%;
+                border: 1px solid #999;
+                text-decoration: none;
+                margin: 0 4px;
+                font-size: 16px;
+                line-height: 15px;
+
+                &.not_applicable {
+                    opacity: 0;
+                    pointer-events: none;
+                }
+            }
+
+            .main {
+                flex: 1;
+                display: flex;
+                align-items: baseline;
+
+                .bid {
+                    font-family: monospace;
+                    padding: 3px 6px;
+                    background: rgba(0,0,0,0.2);
+                    border-radius: 3px;
+                    font-size: 12px;
+                    margin: 0 12px 0 0;
+
+                    &:hover {
+                        .bid-goto {
+                            display: inline-block;
+                            padding: 0 0 0 4px;
+                            font-size: 200%;
+                            line-height: 10px;
+                            vertical-align: sub;
+                            text-decoration: dotted;
+                        }
+                    }
+
+                    .bid-goto {
+                        display: none;
+                    }
+                }
+            }
+
+            .goto-link {
+                margin: 0 8px;
+                display: inline-block;
+                height: 16px;
+                font-size: 90%;
+                border-bottom: 1px dotted white;
+            }
+
+            .status-label {
+                font-family: monospace;
+                padding: 3px 6px;
+                background: rgba(0,0,0,0.2);
+                border-radius: 3px;
+                font-size: 12px;
+                margin: 0 12px 0 0;
+
+                &.deleted {
+                    background: #99999933;
+                }
+                &.failed, &.complete {
+                    background: #99000033;
+                }
+                &.success {
+                    background: #00990033;
+                }
+            }
+
+            .status-block {
+                width: 10em;
+                text-align: center;
+            }
+        }
+
+        > .subitems {
+            padding-left: 16px;
+
+            >.load-more {
+                padding: 4px 0;
+                text-align: center;
+                border-bottom: 1px dashed white;
+                a {
+                    border-bottom: 1px dotted white;
+                    text-decoration: none;
+                }
+            }
+        }
+    }
+}