canvas_sync 0.7.3 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: b7439a3d10f86fcf936b85479a44a4869eac49c32be3d035ab652442ba3ec880
-  data.tar.gz: e1bd2bcc305b7c737bcd5150357319c2feaacf04bceff757888a086a9101f293
+SHA1:
+  metadata.gz: bcfe0a03413b48c2c1ca696c0d60c7aab54b9a8f
+  data.tar.gz: 1c6a2daa99f260e56739463bac62a6ab1d96ed67
 SHA512:
-  metadata.gz: 7fb2641bcaa7c472cfd0078c061be94a360da56c46671917fd5096a18fa86b375cbf3dd26f21fcab2618a88d24110fa94548ab9b29536c8f0411a1784c9a91ca
-  data.tar.gz: cc3bd13b13000947a20dd3cf0b1e5bd3c58733b847e17f19d733ce218f2f973801e9b9a165e4c5667c6f490266ce4727a2c77f691505a7840a39f951de39600c
+  metadata.gz: 9b0b91b958c03a577ed3f5b31f2d70436f8577e13450282dd27818cb001eaf45ad19ad9ca3b70feba7c573fa4dec15ab4837c9a354675445f16ddd90162ca21e
+  data.tar.gz: 4387a495528f967c274fc5c08f51f753639aca0ffa5b6f3550cf59e59cf144f0c1c87a4585352870b99c5ef3034d1872b12ee34c4800b094960948677d497c97

data/README.md

@@ -229,6 +229,85 @@ Sidekiq.configure_server do |config|
 end
 ```
 
+## Syncronize different reports
+CanvasSync provides the functionality to import data from other reports into an specific table.
+
+This can be achived by using the followin method
+
+```ruby
+CanvasSync.provisioning_sync(<array of models to sync>, term_scope: <optional term scope>)
+CanvasSync
+  .simple_report_sync(
+    [
+      {
+        report_name: <report name>,
+        model: <model to sync>,
+        params: <hash with the require parameters the report needs to sync>
+      },
+      {
+        report_name: <report name>,
+        model: <model to sync>,
+        params: <hash with the require parameters the report needs to sync>
+      },
+      ...
+    ],
+    term_scope: <optional term scope>
+  )
+```
+
+Example:
+
+```ruby
+CanvasSync
+  .simple_report_sync(
+    [
+      {
+        report_name: 'proservices_provisioning_csv',
+        model: 'users',
+        params: {
+          "parameters[include_deleted]" => true,
+          "parameters[users]" => true
+        }
+      },
+      {
+        report_name: 'proservices_provisioning_csv',
+        model: 'accounts',
+        params: {
+          "parameters[include_deleted]" => true,
+          "parameters[accounts]" => true
+        }
+      }
+    ]
+  )
+```
+
+Example with the term_scope active:
+
+```ruby
+CanvasSync
+  .simple_report_sync(
+    [
+      {
+        report_name: 'proservices_provisioning_csv',
+        model: 'sections',
+        params: {
+          "parameters[include_deleted]" => true,
+          "parameters[sections]" => true
+        }
+      },
+      {
+        report_name: 'proservices_provisioning_csv',
+        model: 'courses',
+        params: {
+          "parameters[include_deleted]" => true,
+          "parameters[courses]" => true
+        }
+      }
+    ],
+    term_scope: 'active'
+  )
+```
+
 ## Configuration
 
 You can configure CanvasSync settings by doing the following:

data/lib/canvas_sync.rb

@@ -7,6 +7,7 @@ require "canvas_sync/jobs/report_starter"
 require "canvas_sync/jobs/report_checker"
 require "canvas_sync/jobs/report_processor_job"
 require "canvas_sync/jobs/sync_provisioning_report_job"
+require "canvas_sync/jobs/sync_simple_table_job.rb"
 require "canvas_sync/jobs/sync_assignments_job"
 require "canvas_sync/jobs/sync_submissions_job"
 require "canvas_sync/jobs/sync_assignment_groups_job"
@@ -52,6 +53,10 @@ module CanvasSync
     course_section
   ].freeze
 
+  SUPPORTED_NON_PROV_REPORTS = %w[
+    graded_submissions
+  ].freeze
+
   class << self
     # Runs a standard provisioning sync job with no extra report types.
     # Terms will be synced first using the API. If you are syncing users/roles/admins
@@ -72,6 +77,19 @@ module CanvasSync
       invoke_next(default_provisioning_report_chain(models, term_scope, legacy_support, account_id))
     end
 
+    # Runs a report different from provisioning sync job with no extra report types.
+    #
+    # @param reports_mapping [Array<Hash>] An Array of hash that list the model and params with their report you
+    #   want to import:
+    #   [{model: 'submissions', report_name: 'my_report_name_csv', params: { "parameters[include_deleted]" => true } }, ...]
+    # @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model.
+    #   The provisioning report will be run for each of the terms contained in that scope.
+    # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
+    #   canvas_sync_client methods require an account ID.
+    def simple_report_sync(reports_mapping, term_scope: nil, account_id: nil)
+      invoke_next(simple_report_chain(reports_mapping, term_scope, account_id))
+    end
+
     # Runs a chain of ordered jobs
     #
     # See the README for usage and examples
@@ -99,6 +117,33 @@ module CanvasSync
       next_job_class.perform_later(duped_job_chain, next_job[:options])
     end
 
+    # Syn any report to an specific set of models
+    #
+    # @param reports_mapping [Array<Hash>] List of reports with their specific model and params
+    # @param term_scope [String]
+    # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
+    #   canvas_sync_client methods require an account ID.
+    def simple_report_chain(reports_mapping, term_scope=nil, account_id=nil)
+      jobs = reports_mapping.map do |report|
+        {
+          job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
+          options: {
+            report_name: report[:report_name],
+            model: report[:model],
+            mapping: report[:model],
+            klass: report[:model].singularize.capitalize.to_s,
+            term_scope: term_scope,
+            params: report[:params]
+          }
+        }
+      end
+
+      global_options = {}
+      global_options[:account_id] = account_id if account_id.present?
+
+      { jobs: jobs, global_options: global_options }
+    end
+
     # Syncs terms, users/roles/admins if necessary, then the rest of the specified models.
     #
     # @param models [Array<String>]

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -22,13 +22,16 @@ module CanvasSync
         database_column_names = mapping.values.map { |value| value[:database_column_name] }
         rows = []
         row_ids = {}
+        database_conflict_column_name = conflict_target ? mapping[conflict_target][:database_column_name] : nil
 
         CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
           row = yield(row) if block_given?
           next if row.nil?
 
-          next if row_ids[row[conflict_target]]
-          row_ids[row[conflict_target]] = true
+          if conflict_target
+            next if row_ids[row[conflict_target]]
+            row_ids[row[conflict_target]] = true
+          end
 
           rows << csv_column_names.map do |column|
             if mapping[column][:type].to_sym == :datetime
@@ -41,24 +44,26 @@ module CanvasSync
           end
 
           if rows.length >= batch_size
-            perform_import(klass, database_column_names, rows, mapping[conflict_target][:database_column_name], import_args)
+            perform_import(klass, database_column_names, rows, database_conflict_column_name, import_args)
             rows = []
             row_ids = {}
           end
         end
 
-        perform_import(klass, database_column_names, rows, mapping[conflict_target][:database_column_name], import_args)
+        perform_import(klass, database_column_names, rows, database_conflict_column_name, import_args)
       end
 
       def self.perform_import(klass, columns, rows, conflict_target, import_args={})
         return if rows.length.zero?
         columns = columns.dup
 
-        options = { validate: false, on_duplicate_key_update: {
-          conflict_target: conflict_target,
+        update_conditions = {
           condition: condition_sql(klass, columns),
           columns: columns
-        } }.merge(import_args)
+        }
+        update_conditions[:conflict_target] = conflict_target if conflict_target
+
+        options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
 
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
         klass.import(columns, rows, options)

data/lib/canvas_sync/jobs/sync_simple_table_job.rb

@@ -0,0 +1,41 @@
+module CanvasSync
+  module Jobs
+    class SyncSimpleTableJob < ReportStarter
+      #
+      # Starts a report processor for the specified report
+      # (the specified report must be enabled)
+      #
+      # @param job_chain [Hash]
+      # @param options [Hash]
+      def perform(job_chain, options)
+        if options[:term_scope]
+          Term.send(options[:term_scope]).find_each do |term|
+            # Deep copy the job_chain so each report gets the correct term id passed into
+            # its options with no side effects
+            duped_job_chain = Marshal.load(Marshal.dump(job_chain))
+            duped_job_chain[:global_options][:canvas_term_id] = term.canvas_term_id
+            start_report(report_params(options, term.canvas_term_id), duped_job_chain, options)
+          end
+        else
+          start_report(report_params(options), job_chain, options)
+        end
+      end
+
+      def start_report(params, job_chain, options)
+        CanvasSync::Jobs::ReportStarter.perform_later(
+          job_chain,
+          options[:report_name],
+          params,
+          CanvasSync::Processors::NormalProcessor.to_s,
+          options,
+        )
+      end
+
+      def report_params(options, canvas_term_id=nil)
+        params = options[:params] || {}
+        params["parameters[enrollment_term_id]"] = canvas_term_id if canvas_term_id
+        params
+      end
+    end
+  end
+end

data/lib/canvas_sync/processors/normal_processor.rb

@@ -0,0 +1,26 @@
+require_relative "./report_processor"
+
+module CanvasSync
+  module Processors
+    # Processes any report using the normal bulk importer.
+    #
+    # @param report_file_path [String]
+    # @param options [Hash]
+    # @param report_id [Integer]
+    class NormalProcessor < ReportProcessor
+      def self.process(report_file_path, options, report_id)
+        new(report_file_path, options)
+      end
+
+      def initialize(report_file_path, options)
+        conflict_target =  mapping[options[:mapping].to_sym][:conflict_target]
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          mapping[options[:mapping].to_sym][:report_columns],
+          options[:klass].constantize,
+          conflict_target ? conflict_target.to_sym : conflict_target
+        )
+      end
+    end
+  end
+end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.7.3".freeze
+  VERSION = "0.8.0".freeze
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -153,4 +153,81 @@ RSpec.describe CanvasSync do
       end
     end
   end
+
+  describe ".simple_report_sync" do
+
+    it 'invokes the first job in the queue and passes on the rest of the job chain' do
+      expected_job_chain = CanvasSync.simple_report_chain(
+        [
+          {
+            report_name: 'proservices_provisioning_csv',
+            model: 'users',
+            params: {
+              "parameters[include_deleted]" => true,
+              "parameters[users]" => true
+            }
+          }
+        ]
+      )
+      first_job = expected_job_chain[:jobs].shift
+
+      expect(CanvasSync::Jobs::SyncSimpleTableJob).to receive(:perform_later)
+        .with(
+          expected_job_chain,
+          first_job[:options]
+        )
+
+      CanvasSync.simple_report_sync(
+        [
+          {
+            report_name: 'proservices_provisioning_csv',
+            model: 'users',
+            params: {
+              "parameters[include_deleted]" => true,
+              "parameters[users]" => true
+            }
+          }
+        ]
+      )
+    end
+
+    it 'receives the job chain for the specified table' do
+      chain = CanvasSync.simple_report_chain(
+        [
+          {
+            report_name: 'proservices_provisioning_csv',
+            model: 'users',
+            params: {
+              "parameters[include_deleted]" => true,
+              "parameters[users]" => true
+            }
+          }
+        ]
+      )
+
+      expected_job_chain = {
+        jobs: [
+          {
+            job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
+            options: {
+              report_name: 'proservices_provisioning_csv',
+              model: 'users',
+              mapping: 'users',
+              klass: 'User',
+              term_scope: nil,
+              params: {
+                "parameters[include_deleted]" => true,
+                "parameters[users]" => true
+              }
+            }
+          }
+        ],
+        global_options: {}
+      }
+
+      expect(chain).to eq(expected_job_chain)
+
+    end
+  end
+  
 end

data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::Jobs::SyncSimpleTableJob do
+  describe '#perform' do
+    let(:job_chain) { {jobs: [], global_options: {}} }
+
+    context 'Simple report' do
+      let!(:active_term_1) { FactoryGirl.create(:term) }
+      let!(:inactive_term_1) { FactoryGirl.create(:term, workflow_state: 'inactive') }
+
+      it 'enqueues a ReportStarter for a provisioning report for the specified model for a term' do
+        expected_job_chain = Marshal.load(Marshal.dump(job_chain))
+        expected_job_chain[:global_options][:canvas_term_id] = active_term_1.canvas_term_id
+
+        expect(CanvasSync::Jobs::ReportStarter).to receive(:perform_later)
+          .with(
+            expected_job_chain,
+            'proservices_provisioning_csv',
+            {
+              "parameters[include_deleted]" => true,
+              "parameters[courses]" => true,
+              "parameters[enrollment_term_id]" => active_term_1.canvas_term_id
+            },
+            CanvasSync::Processors::NormalProcessor.to_s,
+            {
+              report_name: 'proservices_provisioning_csv',
+              model: 'courses',
+              mapping: 'courses',
+              klass: 'Course',
+              term_scope: 'active',
+              params: {
+                "parameters[include_deleted]" => true,
+                "parameters[courses]" => true,
+                "parameters[enrollment_term_id]" => active_term_1.canvas_term_id
+              }
+            }
+          )
+
+        CanvasSync::Jobs::SyncSimpleTableJob.perform_now(
+          job_chain,
+          {
+            report_name: 'proservices_provisioning_csv',
+            model: 'courses',
+            mapping: 'courses',
+            klass: 'Course',
+            term_scope: 'active',
+            params: {
+              "parameters[include_deleted]" => true,
+              "parameters[courses]" => true
+            }
+          }
+        )
+
+      end
+    end
+  end
+end

data/spec/canvas_sync/processors/normal_processor_spec.rb

@@ -0,0 +1,27 @@
+require "spec_helper"
+
+RSpec.describe CanvasSync::Processors::NormalProcessor do
+  let(:subject) { CanvasSync::Processors::NormalProcessor }
+
+  describe "#process" do
+    it "inserts row" do
+      expect {
+        subject.process(
+          "spec/support/fixtures/reports/courses.csv",
+          {
+            report_name: 'proservices_provisioning_csv',
+            model: 'courses',
+            mapping: 'courses',
+            klass: 'Course',
+            term_scope: 'active',
+            params: {
+              "parameters[include_deleted]" => true,
+              "parameters[courses]" => true
+            }
+          },
+          1
+        )
+      }.to change { Course.count }.by(2)
+    end
+  end
+end