inst_data_shipper 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc59b5931e2a448fa9a687147b848800fd4478da037762eb3075f3e9955706d3
-  data.tar.gz: 13382736ae30290d0997af560ce0c6b0e5445dbb6ddbbf832368fd495e467dca
+  metadata.gz: d2f09c00768c10bfc79fb35c689131740739a8889c81ee02186f4bd9e70b446b
+  data.tar.gz: 9550a780011a5fa988fff785429a637175a16634911186e7bb7625e80ac09141
 SHA512:
-  metadata.gz: 3a118fc9c812be9b61c4ddfd7506a8f4c6b0c1d1ab0ef7deef3d45e13ee35631f110283eca1d95c2ce2c817e2c8144735d716380d9bbaf07e00516913d73ec31
-  data.tar.gz: c103956f29f5a27c6039c4eb8381d9a42a0fe54bd1c9df8e0b86daff32f7f768c61f255ab8819b1b0e197b895becc0fbf9ce55a3be8072f69e6597aef4416e6b
+  metadata.gz: f5f4fd607175698b654476ed4ef50d600be06b380071469e9fda824433d3d7014bacbecc53cb25238a5ade8d1ce5b302192bc7aa8f48c922e0ae029b56d4fcac
+  data.tar.gz: 6d069aa519374716b5690ba2428f6f4ec2000ae36b716921e36aab9160d126dc53a4db1f8d7614d97d82849d62138d7464ffd4f719c39ee226f7becad7b50514

data/app/models/inst_data_shipper/dump_batch.rb

@@ -1,7 +1,5 @@
 module InstDataShipper
   class DumpBatch < ApplicationRecord
-    serialize :job_arguments, Array
-
     ERROR_STATUS = "error".freeze
     SUCCESS_STATUS = "success".freeze
     ENQUEUED_STATUS = "enqueued".freeze

data/db/migrate/20240301090836_create_inst_data_shipper_dump_batches.rb

@@ -11,8 +11,6 @@ class CreateInstDataShipperDumpBatches < CanvasSync::MiscHelper::MigrationClass
 
       t.string :exception
       t.text :backtrace
-      # t.text :metadata
-      # t.text :job_arguments
 
       t.timestamps
     end

data/lib/inst_data_shipper/data_sources/canvas_reports.rb

@@ -42,7 +42,11 @@ module InstDataShipper
           term.is_a?(Term) ? term.canvas_id : term
         end
 
-        table_def = lookup_table_schema!(kwargs[:schema_name], report_name)
+        table_def = lookup_table_schema(kwargs[:schema_name], report_name) || {}
+        if kwargs[:incremental_config]
+          table_def = table_def.dup
+          table_def[:incremental] = kwargs.delete(:incremental_config)
+        end
 
         _resolve_report_incremenal_parameters(table_def, params)
 
@@ -97,19 +101,51 @@ module InstDataShipper
       end
 
       def _process_canvas_report(report:, schema_name: nil)
-        table_def = lookup_table_schema!(schema_name, report[:report])
+        file_path = "#{working_dir}/temp_report"
+        IO.copy_stream(URI.parse(report['attachment']['url']).open, file_path)
 
-        IO.copy_stream(URI.parse(report['attachment']['url']).open, "#{working_dir}/temp_report.csv")
+        if report['attachment']['content-type'] == 'application/zip'
+          unzipped_file_path = "#{file_path}_unzipped"
 
-        inner_block = ->(file) {
-          CSV.foreach("#{working_dir}/temp_report.csv", headers: true) do |m|
-            file << table_def[:columns].map do |c|
-              instance_exec(m, &c[:block])
+          Zip::File.open(file_path) do |zip_file|
+            zip_file.each do |f|
+              f_path = File.join(unzipped_file_path, f.name)
+              FileUtils.mkdir_p(File.dirname(f_path))
+              zip_file.extract(f, f_path) unless File.exist?(f_path)
             end
           end
-        }
 
-        upload_data(table_def, extra: report['id'], &inner_block)
+          Dir[unzipped_file_path + "/*.csv"].sort.each do |file_path|
+            file_name = file_path.split("/").last.split(".").first
+
+            # If a schema_name Hash is provided, only process files that are explictly mapped
+            next if schema_name && !schema_name[file_name]
+
+            table_def = lookup_table_schema!(schema_name&.[](file_name), report[:report])
+
+            inner_block = ->(file) {
+              CSV.foreach(file_path, headers: true) do |m|
+                file << table_def[:columns].map do |c|
+                  instance_exec(m, &c[:block])
+                end
+              end
+            }
+
+            upload_data(table_def, extra: report['id'], &inner_block)
+          end
+        else
+          table_def = lookup_table_schema!(schema_name, report[:report])
+
+          inner_block = ->(file) {
+            CSV.foreach(file_path, headers: true) do |m|
+              file << table_def[:columns].map do |c|
+                instance_exec(m, &c[:block])
+              end
+            end
+          }
+
+          upload_data(table_def, extra: report['id'], &inner_block)
+        end
       end
 
       def _resolve_report_incremenal_parameters(table_def, params)

data/lib/inst_data_shipper/version.rb

@@ -1,3 +1,3 @@
 module InstDataShipper
-  VERSION = "0.2.3".freeze
+  VERSION = "0.2.4".freeze
 end

data/spec/dummy/tmp/local_secret.txt

@@ -0,0 +1 @@
+6ddf19e3f9f936190c3c79bc546382849bcf6efb4205cbe13797b683ec9a3dbc02cb87e348121c3502809d9475683c26b12c92d973d571953ebb50b0c4cc3641

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: inst_data_shipper
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Instructure CustomDev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-27 00:00:00.000000000 Z
+date: 2024-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -378,6 +378,7 @@ files:
 - spec/dummy/config/routes.rb
 - spec/dummy/config/secrets.yml
 - spec/dummy/db/schema.rb
+- spec/dummy/tmp/local_secret.txt
 - spec/inst_data_shipper/destinations/hosted_data_spec.rb
 - spec/inst_data_shipper/dumper_spec.rb
 - spec/spec_helper.rb
@@ -424,6 +425,7 @@ test_files:
 - spec/dummy/config/secrets.yml
 - spec/dummy/config.ru
 - spec/dummy/db/schema.rb
+- spec/dummy/tmp/local_secret.txt
 - spec/inst_data_shipper/destinations/hosted_data_spec.rb
 - spec/inst_data_shipper/dumper_spec.rb
 - spec/spec_helper.rb