inst_data_shipper 0.2.3 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc59b5931e2a448fa9a687147b848800fd4478da037762eb3075f3e9955706d3
-  data.tar.gz: 13382736ae30290d0997af560ce0c6b0e5445dbb6ddbbf832368fd495e467dca
+  metadata.gz: d40cd83e6acf9255bfae93a00c7b31d8b9f71d3691d13437244165fd89a1ddc5
+  data.tar.gz: 8c4c28ff0ca83b71b35c1f55686eb173ccfd7b264db7d83fe3ade5976ddb79ba
 SHA512:
-  metadata.gz: 3a118fc9c812be9b61c4ddfd7506a8f4c6b0c1d1ab0ef7deef3d45e13ee35631f110283eca1d95c2ce2c817e2c8144735d716380d9bbaf07e00516913d73ec31
-  data.tar.gz: c103956f29f5a27c6039c4eb8381d9a42a0fe54bd1c9df8e0b86daff32f7f768c61f255ab8819b1b0e197b895becc0fbf9ce55a3be8072f69e6597aef4416e6b
+  metadata.gz: 6edb9ebdd4367e6601a5b9920ecd7d0df11ea93b572becb68c42fa63b2c0d78b2ddab488a88f58eb0ad3e4630a011410001d5407aa0c81ce1e8f1eb0f8276bd5
+  data.tar.gz: 66ed5c81ed3b8c4f688c0e6051e376ceb28b370393189aaa1507e5ae6922c36dd130efbaed46c8ae6f132895b345e9661a195803684571ccf03a434ebcbb70b3

data/app/models/inst_data_shipper/dump_batch.rb

@@ -1,7 +1,5 @@
 module InstDataShipper
   class DumpBatch < ApplicationRecord
-    serialize :job_arguments, Array
-
     ERROR_STATUS = "error".freeze
     SUCCESS_STATUS = "success".freeze
     ENQUEUED_STATUS = "enqueued".freeze

data/db/migrate/20240301090836_create_inst_data_shipper_dump_batches.rb

@@ -11,8 +11,6 @@ class CreateInstDataShipperDumpBatches < CanvasSync::MiscHelper::MigrationClass
 
       t.string :exception
       t.text :backtrace
-      # t.text :metadata
-      # t.text :job_arguments
 
       t.timestamps
     end

data/lib/inst_data_shipper/data_sources/canvas_reports.rb

@@ -28,7 +28,7 @@ module InstDataShipper
       end
 
       def import_canvas_report_by_terms(*args, **kwargs)
-        _in_canvas_report_pool(:_import_canvas_report_by_terms, *args, **kwargs)
+        delayed(:_import_canvas_report_by_terms, *args, **kwargs)
       end
 
       def import_existing_report(report, **kwargs)
@@ -42,7 +42,11 @@ module InstDataShipper
           term.is_a?(Term) ? term.canvas_id : term
         end
 
-        table_def = lookup_table_schema!(kwargs[:schema_name], report_name)
+        table_def = lookup_table_schema(kwargs[:schema_name], report_name) || {}
+        if kwargs[:incremental_config]
+          table_def = table_def.dup
+          table_def[:incremental] = kwargs.delete(:incremental_config)
+        end
 
         _resolve_report_incremenal_parameters(table_def, params)
 
@@ -92,24 +96,55 @@ module InstDataShipper
       end
 
       def _in_canvas_report_pool(mthd, *args, **kwargs)
-        pool = CanvasSync::JobBatches::Pool.from_pid(batch_context[:report_processor_pool])
-        Jobs::AsyncCaller.call_from_pool(pool, self.class, mthd, *args, **kwargs)
+        call_in_pool(batch_context[:report_processor_pool], mthd, *args, **kwargs)
       end
 
       def _process_canvas_report(report:, schema_name: nil)
-        table_def = lookup_table_schema!(schema_name, report[:report])
+        file_path = "#{working_dir}/temp_report"
+        IO.copy_stream(URI.parse(report['attachment']['url']).open, file_path)
 
-        IO.copy_stream(URI.parse(report['attachment']['url']).open, "#{working_dir}/temp_report.csv")
+        if report['attachment']['content-type'] == 'application/zip'
+          unzipped_file_path = "#{file_path}_unzipped"
 
-        inner_block = ->(file) {
-          CSV.foreach("#{working_dir}/temp_report.csv", headers: true) do |m|
-            file << table_def[:columns].map do |c|
-              instance_exec(m, &c[:block])
+          Zip::File.open(file_path) do |zip_file|
+            zip_file.each do |f|
+              f_path = File.join(unzipped_file_path, f.name)
+              FileUtils.mkdir_p(File.dirname(f_path))
+              zip_file.extract(f, f_path) unless File.exist?(f_path)
             end
           end
-        }
 
-        upload_data(table_def, extra: report['id'], &inner_block)
+          Dir[unzipped_file_path + "/*.csv"].sort.each do |file_path|
+            file_name = file_path.split("/").last.split(".").first
+
+            # If a schema_name Hash is provided, only process files that are explictly mapped
+            next if schema_name && !schema_name[file_name]
+
+            table_def = lookup_table_schema!(schema_name&.[](file_name), report[:report])
+
+            inner_block = ->(file) {
+              CSV.foreach(file_path, headers: true) do |m|
+                file << table_def[:columns].map do |c|
+                  instance_exec(m, &c[:block])
+                end
+              end
+            }
+
+            upload_data(table_def, extra: report['id'], &inner_block)
+          end
+        else
+          table_def = lookup_table_schema!(schema_name, report[:report])
+
+          inner_block = ->(file) {
+            CSV.foreach(file_path, headers: true) do |m|
+              file << table_def[:columns].map do |c|
+                instance_exec(m, &c[:block])
+              end
+            end
+          }
+
+          upload_data(table_def, extra: report['id'], &inner_block)
+        end
       end
 
       def _resolve_report_incremenal_parameters(table_def, params)

data/lib/inst_data_shipper/dumper.rb

@@ -308,6 +308,11 @@ module InstDataShipper
       Jobs::AsyncCaller.perform_later(self.class.to_s, mthd.to_s, *args, **kwargs)
     end
 
+    def call_in_pool(pool, mthd, *args, **kwargs)
+      pool = CanvasSync::JobBatches::Pool.from_pid(pool) if pool.is_a?(String)
+      Jobs::AsyncCaller.call_from_pool(pool, self.class, mthd, *args, **kwargs)
+    end
+
     delegate :working_dir, to: :executor
 
     def batch

data/lib/inst_data_shipper/version.rb

@@ -1,3 +1,3 @@
 module InstDataShipper
-  VERSION = "0.2.3".freeze
+  VERSION = "0.2.5".freeze
 end

data/spec/dummy/tmp/local_secret.txt

@@ -0,0 +1 @@
+6ddf19e3f9f936190c3c79bc546382849bcf6efb4205cbe13797b683ec9a3dbc02cb87e348121c3502809d9475683c26b12c92d973d571953ebb50b0c4cc3641

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: inst_data_shipper
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.5
 platform: ruby
 authors:
 - Instructure CustomDev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-27 00:00:00.000000000 Z
+date: 2024-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -378,6 +378,7 @@ files:
 - spec/dummy/config/routes.rb
 - spec/dummy/config/secrets.yml
 - spec/dummy/db/schema.rb
+- spec/dummy/tmp/local_secret.txt
 - spec/inst_data_shipper/destinations/hosted_data_spec.rb
 - spec/inst_data_shipper/dumper_spec.rb
 - spec/spec_helper.rb
@@ -424,6 +425,7 @@ test_files:
 - spec/dummy/config/secrets.yml
 - spec/dummy/config.ru
 - spec/dummy/db/schema.rb
+- spec/dummy/tmp/local_secret.txt
 - spec/inst_data_shipper/destinations/hosted_data_spec.rb
 - spec/inst_data_shipper/dumper_spec.rb
 - spec/spec_helper.rb