inst_data_shipper 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 522ed20f5f574d452bf2af49cf8e7ca28377d86d16c9c89cce0aa02a6d89e025
-  data.tar.gz: 5d0cedf102b079833cbf7f670023ab700beb557e1902f454c5c709b6e7b4629e
+  metadata.gz: fc59b5931e2a448fa9a687147b848800fd4478da037762eb3075f3e9955706d3
+  data.tar.gz: 13382736ae30290d0997af560ce0c6b0e5445dbb6ddbbf832368fd495e467dca
 SHA512:
-  metadata.gz: 7880a8aa492c216d767af6a7bf87619ea9ce9f2dd6b2eccdc938acae32b6dd69bf756508f0bf365772e909b93b72b9288ebcdf54afec18348b92da60e5867524
-  data.tar.gz: 3d3762b4bbf53380f519b4b77ec23dce500d403b793c3da7eee6fe9290364048fc579e2c3f4fe39b86f46fc7c7f5ebfa716a30225a38f4beb05b40c3fd25b1a5
+  metadata.gz: 3a118fc9c812be9b61c4ddfd7506a8f4c6b0c1d1ab0ef7deef3d45e13ee35631f110283eca1d95c2ce2c817e2c8144735d716380d9bbaf07e00516913d73ec31
+  data.tar.gz: c103956f29f5a27c6039c4eb8381d9a42a0fe54bd1c9df8e0b86daff32f7f768c61f255ab8819b1b0e197b895becc0fbf9ce55a3be8072f69e6597aef4416e6b

data/README.md

@@ -22,6 +22,10 @@ bundle exec rake db:migrate
 
 The main tool provided by this Gem is the `InstDataDumper::Dumper` class. It is used to define a "Dump" which is a combination of tasks and schema.
 
+It is a assumed that a `Dumper` class definition is the source of truth for all tables that it manages, and that no other processes affect the tables' data or schema. You can break this assumption, but you should understand how the `incremental` logic works and what will and will not trigger a full table upload. Dumpers have a `export_genre` method that determines the what Dumps to look at when calculating incrementals.
+- High level, the HD backend will look for a past dump of the same genre. If not found, a full upload of all tables is triggered. If found, each table's schema is compared; any tables with mismatched schema (determined by hashing) will do a full upload.
+- Note that `Proc`s in the schema are not included in the hash calculation. If you change a `Proc` implementation and need to trigger a full-upload of the table, you'll need to change something else too (like the `version`).
+
 Here is an example `Dumper` implementation, wrapped in an ActiveJob job:
 ```ruby
 class HostedDataPushJob < ApplicationJob
@@ -55,27 +59,31 @@ class HostedDataPushJob < ApplicationJob
       #  `if:` may be a Proc or a Symbol (of a method on the Dumper)
       incremental "updated_at", on: [:id], if: ->() {}
 
-      # Schema's may declaratively define the data source.
+      # Schemas may declaratively define the data source.
       # This can be used for basic schemas where there's a 1:1 mapping between source table and destination table, and there is no conditional logic that needs to be performed.
       # In order to apply these statements, your Dumper must call `auto_enqueue_from_schema`.
       source :local_table
       # A Proc can also be passed. The below is equivalent to the above
       source ->(table_def) { import_local_table(table_def[:model] || table_def[:warehouse_name]) }
 
+      # You may manually note a version on the table.
+      # Note that if a version is present, the version value replaces the hash-comparison when calculating incrementals, so you must change the version whenever the schema changes enough to trigger a full-upload
+      version "1.0.0"
+
       column :name_in_destinations, :maybe_optional_sql_type, "Optional description of column"
 
       # The type may usually be omitted if the `table()` is passed a Model class, but strings are an exception to this
-      custom_column :name, :"varchar(128)"
+      column :name, :"varchar(128)"
 
       # `from:` May be...
       # A Symbol of a method to be called on the record
-      custom_column :sis_type, :"varchar(32)", from: :some_model_method
+      column :sis_type, :"varchar(32)", from: :some_model_method
       # A String of a column to read from the record
-      custom_column :sis_type, :"varchar(32)", from: "sis_source_type"
+      column :sis_type, :"varchar(32)", from: "sis_source_type"
       # A Proc to be called with each record
-      custom_column :sis_type, :"varchar(32)", from: ->(rec) { ... }
+      column :sis_type, :"varchar(32)", from: ->(rec) { ... }
       # Not specified. Will default to using the Schema Column Name as a String ("sis_type" in this case)
-      custom_column :sis_type, :"varchar(32)"
+      column :sis_type, :"varchar(32)"
     end
 
     table("my_table", model: ALocalModel) do

data/lib/inst_data_shipper/basic_dumper.rb

@@ -1,12 +1,12 @@
 module InstDataShipper
   class BasicDumper < Dumper
-    def self.perform_dump(destinations:, schema:, &block)
+    def self.perform_dump(destinations:, schema:, force_full_tables: nil, &block)
       raise "Schema must be a constantizable string" unless schema.is_a?(String)
 
       dumper = new(destinations)
       dumper.instance_variable_set(:@schema_pointer, schema)
       dumper.instance_variable_set(:@body_block, block)
-      dumper.begin_dump
+      dumper.begin_dump(force_full_tables: force_full_tables)
 
       dumper.tracker
     end

data/lib/inst_data_shipper/destinations/base.rb

@@ -63,6 +63,7 @@ module InstDataShipper
 
       protected
 
+      # TODO Make a class-method?
       def parse_configuration(uri)
         if block_given?
           parsed = URI.parse(uri)

data/lib/inst_data_shipper/destinations/hosted_data.rb

@@ -10,20 +10,34 @@ module InstDataShipper
           begin
             last_dump = hosted_data_client.get("api/v1/custom_dumps/last", {
               status: 'imported',
-              # schema_version: convert_schema[:version],
               tags: [
-                "ids-schema=#{dumper.schema_digest}",
+                # We could also include app in the filter, but each app should already have a distinct key in HD
                 "ids-genre=#{dumper.export_genre}",
               ],
+              include: ["schema"],
             }).body.with_indifferent_access
 
             if last_dump[:created_at] < context[:incremental_since]
               InstDataShipper.logger.info("Last successful HostedData dump is older than incremental_since - bumping back incremental_since")
               context[:incremental_since] = last_dump[:created_at]
             end
+
+            if (hd_tables = last_dump[:schema]).present?
+              metadatas = hd_tables.values.map { |t| t[:ids_meta] }.compact.map { |t| [t[:table_warehouse_name], t] }.to_h
+              schema[:tables].each do |ts|
+                hd_meta = metadatas[ts[:warehouse_name]]
+
+                if !dumper.table_schema_compatible?(ts, hd_meta)
+                  InstDataShipper.logger.info("Last successful HostedData dump of #{ts[:warehouse_name]} has a different schema - forcing full table")
+                  context[:force_full_tables] << ts[:warehouse_name]
+                end
+              end
+            elsif !last_dump[:tags].include?("ids-schema=#{dumper.schema_digest}")
+              InstDataShipper.logger.info("Last successful HostedData dump of the same genre has a different schema - not using incremental_since")
+              context[:incremental_since] = nil
+            end
           rescue Faraday::ResourceNotFound
-            # TODO It'd be nice to make this per-table
-            InstDataShipper.logger.info("No Last successful HostedData dump of the same schema - not using incremental_since")
+            InstDataShipper.logger.info("No Last successful HostedData dump of the same genre - not using incremental_since")
             context[:incremental_since] = nil
           end
         end
@@ -115,6 +129,7 @@ module InstDataShipper
                 type: coltype,
               }
             end,
+            ids_meta: dumper.table_schema_metadata(ts),
           }
         end
 

data/lib/inst_data_shipper/dumper.rb

@@ -5,18 +5,18 @@ module InstDataShipper
     define_hook :initialize_dump_batch
     define_hook :finalize_dump_batch
 
-    def self.perform_dump(destinations)
+    def self.perform_dump(destinations, force_full_tables: nil)
       raise "Must subclass Dumper to use perform_dump" if self == Dumper
 
       dumper = new(destinations)
-      dumper.begin_dump
+      dumper.begin_dump(force_full_tables: force_full_tables)
 
       dumper.tracker
     end
 
     def self.define(include: [], schema: , &blk)
       Class.new(self) do
-        include(*include)
+        include(*include) if include.present?
 
         if blk.nil? && schema[:tables].any? { |t| t[:sourcer].present? }
           blk = -> { auto_enqueue_from_schema }
@@ -51,7 +51,7 @@ module InstDataShipper
 
     public
 
-    def begin_dump
+    def begin_dump(force_full_tables: nil)
       raise "Dump already begun" unless @raw_destinations.present?
 
       @tracker = tracker = DumpBatch.create(job_class: self.class.to_s, genre: export_genre, status: 'in_progress')
@@ -59,6 +59,7 @@ module InstDataShipper
       @batch_context = context = {
         # TODO Consider behavior if last is still running
         incremental_since: last_successful_tracker&.created_at,
+        force_full_tables: force_full_tables || [],
       }
 
       destinations.each do |dest|
@@ -79,6 +80,8 @@ module InstDataShipper
           context[:destinations] = @raw_destinations
         end
 
+        context.delete(:force_full_tables) if context[:force_full_tables].empty?
+
         Sidekiq::Batch.new.tap do |batch|
           context[:root_bid] = batch.bid
           tracker.update(batch_id: batch.bid)
@@ -135,9 +138,42 @@ module InstDataShipper
       Digest::MD5.hexdigest(schema.to_json)[0...8]
     end
 
+    def table_schema_metadata(table_def)
+      meta = {
+        table_warehouse_name: table_def[:warehouse_name],
+        table_schema_hash: table_schema_hash(table_def),
+      }
+
+      meta[:table_schema_version] = table_def[:version] if table_def[:version].present?
+
+      meta
+    end
+
+    def table_schema_compatible?(table_def, meta_hash)
+      # Force full-table-upload if:
+      # - The table is not present in the last dump
+      return false unless meta_hash
+
+      # - The table's explicitly-set versions do not match
+      return false if meta_hash[:table_schema_version] != table_def[:version]
+
+      # - The table does not have an explicitly-set version and the schema hash does not match
+      return false if !table_def[:version].present? && meta_hash[:table_schema_hash] != table_schema_hash(table_def)
+
+      true
+    end
+
+    def table_schema_hash(table_def)
+      Digest::MD5.hexdigest(table_def.to_json)[0...8]
+    end
+
     def table_is_incremental?(table_def)
       return false unless incremental_since.present?
 
+      table_def = lookup_table_schema!(table_def) if table_def.is_a?(String)
+
+      return false if batch_context[:force_full_tables]&.include?(table_def[:warehouse_name])
+
       # TODO Return false if table's schema changes
       if (inc = table_def[:incremental]).present?
         differ = inc[:if]
@@ -192,11 +228,16 @@ module InstDataShipper
       raise NotImplementedError
     end
 
+    def enqueue_table_from_schema(table_def)
+      table_def = lookup_table_schema!(table_def) if table_def.is_a?(String)
+      instance_exec(table_def, &table_def[:sourcer])
+    end
+
     def auto_enqueue_from_schema
       schema[:tables].each do |table_def|
         src = table_def[:sourcer]
         next unless src.present?
-        instance_exec(table_def, &src)
+        enqueue_table_from_schema(table_def)
       end
     end
 

data/lib/inst_data_shipper/schema_builder.rb

@@ -1,5 +1,5 @@
 module InstDataShipper
-  # This class ends up fill two roles - Schema and Mapping.
+  # This class ends up filling two roles - Schema and Mapping.
   # It makes for a clean API, but it's a little less canonical since, (eg) the S3 destination doesn't need column type annotations.
   class SchemaBuilder
     attr_reader :schema
@@ -92,15 +92,17 @@ module InstDataShipper
         }
       end
 
-      def source(source, override_model=nil, **kwargs)
+      def source(source=nil, override_model=nil, **kwargs, &block)
         raise "Source already set" if options[:sourcer].present?
+        raise ArgumentError, "Cannot provide source and block" if source.present? && block.present?
 
         if source.is_a?(Symbol)
           mthd = :"import_#{source}"
           options = self.options
           source = ->(table_def) { send(mthd, override_model || options[:model] || options[:warehouse_name], schema_name: options[:warehouse_name], **kwargs) }
         end
-        options[:sourcer] = source
+
+        self.options[:sourcer] = source
       end
 
       def column(name, *args, refs: [], from: nil, **extra, &block)

data/lib/inst_data_shipper/version.rb

@@ -1,3 +1,3 @@
 module InstDataShipper
-  VERSION = "0.2.2".freeze
+  VERSION = "0.2.3".freeze
 end

data/spec/dummy/app/models/application_record.rb

@@ -0,0 +1,3 @@
+class ApplicationRecord < ActiveRecord::Base
+  self.abstract_class = true
+end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_10_30_210836) do
+ActiveRecord::Schema.define(version: 2024_03_01_090836) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -42,4 +42,17 @@ ActiveRecord::Schema.define(version: 2020_10_30_210836) do
     t.string "batch_bid"
   end
 
+  create_table "inst_data_shipper_dump_batches", id: :serial, force: :cascade do |t|
+    t.datetime "started_at"
+    t.datetime "completed_at"
+    t.string "status"
+    t.string "job_class"
+    t.string "genre"
+    t.string "batch_id"
+    t.string "exception"
+    t.text "backtrace"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
 end

data/spec/inst_data_shipper/destinations/hosted_data_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+RSpec.describe InstDataShipper::Destinations::HostedData do
+  let(:dumper) { TestDumper.new }
+  let(:dest) { described_class.new("", "hosted-data://key@hosted-data.local", dumper) }
+
+  describe "#preinitialize_dump" do
+    let(:context) { {
+      incremental_since: 1.day.ago,
+      force_full_tables: [],
+    } }
+
+    let(:response_body) { {
+      created_at: 1.hour.ago.iso8601,
+      tags: ["ids-schema=abc123"],
+    } }
+
+    before :each do
+      allow(dumper).to receive(:schema_digest).and_return("abc123")
+      stub_request(:get, "https://hosted-data.local/api/v1/custom_dumps/last").with(query: hash_including()).to_return do |req|
+        {
+          body: response_body.to_json,
+          headers: {'Content-Type' => 'application/json'},
+        }
+      end
+    end
+
+    it "leaves incremental_since alone if last dump is newer" do
+      x = context[:incremental_since]
+      dest.preinitialize_dump(context)
+      expect(context[:incremental_since]).to eq x
+    end
+
+    it "bumps back incremental_since if last dump is older" do
+      response_body[:created_at] = 2.days.ago.iso8601
+      dest.preinitialize_dump(context)
+      expect(context[:incremental_since]).to eq(response_body[:created_at])
+    end
+
+    it "nils incremental_since if last dump is of a different schema" do
+      response_body[:tags] = ["ids-schema=def456"]
+      dest.preinitialize_dump(context)
+      expect(context[:incremental_since]).to be nil
+    end
+
+    context "per-table forcing" do
+      before :each do
+        response_body[:schema] = {
+          "test_table" => {
+            ids_meta: {
+              table_warehouse_name: "test_table",
+              table_schema_hash: "def456",
+            },
+          }
+        }
+      end
+
+      it "does not trigger force-full if the schema hash matches" do
+        allow(dumper).to receive(:table_schema_hash).and_return("def456")
+        dest.preinitialize_dump(context)
+        expect(context[:force_full_tables] || []).to_not include "test_table"
+      end
+
+      it "triggers force-full if the schema hash does not match" do
+        allow(dumper).to receive(:table_schema_hash).and_return("def123")
+        dest.preinitialize_dump(context)
+        expect(context[:force_full_tables]).to include "test_table"
+      end
+
+      it "triggers force-full if versions do not match" do
+        response_body[:schema]["table_with_version"] = { ids_meta: { table_warehouse_name: "table_with_version", table_schema_version: "1.1" } }
+        dest.preinitialize_dump(context)
+        expect(context[:force_full_tables]).to include "table_with_version"
+      end
+
+      it "does not trigger force-full if the versions match" do
+        response_body[:schema]["table_with_version"] = { ids_meta: { table_warehouse_name: "table_with_version", table_schema_version: "1.0" } }
+        dest.preinitialize_dump(context)
+        expect(context[:force_full_tables] || []).to_not include "table_with_version"
+      end
+
+      it "does not consider the schema hash if a version is present" do
+        response_body[:schema]["table_with_version"] = { ids_meta: { table_warehouse_name: "table_with_version", table_schema_version: "1.0", table_schema_hash: "not_matching" } }
+        dest.preinitialize_dump(context)
+        expect(context[:force_full_tables]).to_not include "table_with_version"
+      end
+    end
+  end
+end

data/spec/inst_data_shipper/dumper_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+RSpec.describe InstDataShipper::Dumper do
+  let(:dumper) { TestDumper.new }
+  let(:context) { {
+    incremental_since: 1.day.ago.iso8601,
+  } }
+
+  before :each do
+    allow(dumper).to receive(:batch_context).and_return(context)
+  end
+
+  describe "#auto_enqueue_from_schema" do
+    it "makes the expected calls" do
+      expect(dumper).to receive(:import_local_table).with(InstDataShipper::DumpBatch, schema_name: "local_table_test")
+      dumper.send(:enqueue_table_from_schema, "local_table_test")
+    end
+
+    it "enqueues all tables with a sourcer" do
+      expect(dumper).to receive(:enqueue_table_from_schema).with(hash_including(warehouse_name: "test_table"))
+      expect(dumper).to receive(:enqueue_table_from_schema).with(hash_including(warehouse_name: "local_table_test"))
+      allow(dumper).to receive(:enqueue_table_from_schema)
+      dumper.send(:auto_enqueue_from_schema)
+    end
+
+    it "doesn't try to enqueue a table with no sourcer" do
+      expect(dumper).to_not receive(:enqueue_table_from_schema).with(hash_including(warehouse_name: "incremental_table"))
+      allow(dumper).to receive(:enqueue_table_from_schema)
+      dumper.send(:auto_enqueue_from_schema)
+    end
+  end
+
+  describe "#table_is_incremental?" do
+    it "usually returns as the table defines" do
+      expect(dumper.table_is_incremental?("test_table")).to be false
+      expect(dumper.table_is_incremental?("incremental_table")).to be true
+    end
+
+    it "returns false if incremental_since is nil" do
+      context[:incremental_since] = nil
+      expect(dumper.table_is_incremental?("incremental_table")).to be false
+    end
+
+    it "returns false if table is forced to full" do
+      context[:force_full_tables] = ["incremental_table"]
+      expect(dumper.table_is_incremental?("incremental_table")).to be false
+    end
+
+    it "supports a proc for the if: option" do
+      dumper.instance_variable_set(:@proc_incremental, true)
+      expect(dumper.table_is_incremental?("proc_incremental_table")).to be true
+      dumper.instance_variable_set(:@proc_incremental, false)
+      expect(dumper.table_is_incremental?("proc_incremental_table")).to be false
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -10,7 +10,7 @@ require 'spec_helper'
 require 'factory_bot_rails'
 require 'timecop'
 require 'webmock/rspec'
-require 'support/fake_canvas'
+require 'support/test_dumper'
 require 'shoulda/matchers'
 require 'pry'
 require 'pry-nav'
@@ -18,11 +18,6 @@ require 'with_model'
 
 require 'sidekiq/testing'
 Sidekiq::Testing.fake!
-Sidekiq::Testing.server_middleware do |chain|
-  chain.add CanvasSync::JobBatches::Sidekiq::ServerMiddleware
-end
-
-Dir[File.dirname(__FILE__) + "/job_batching/support/**/*.rb"].each {|f| require f }
 
 ActiveRecord::Migration.maintain_test_schema!
 
@@ -43,10 +38,6 @@ RSpec.configure do |config|
   config.expect_with :rspec do |c|
     c.syntax = :expect
   end
-
-  config.before(:each) do
-    stub_request(:any, /test.instructure.com/).to_rack(FakeCanvas)
-  end
 end
 
 Shoulda::Matchers.configure do |config|

data/spec/support/test_dumper.rb

@@ -0,0 +1,31 @@
+
+TestSchema = InstDataShipper::SchemaBuilder.build do
+  table("test_table", model: "Account") do
+    source ->(tdef) {}
+
+    column :id, :"varchar(32)", from: "Id"
+  end
+
+  table("table_with_version", model: "Account") do
+    version "1.0"
+    column :id
+  end
+
+  table("local_table_test", model: InstDataShipper::DumpBatch) do
+    source :local_table
+
+    column :id
+  end
+
+  table("incremental_table") do
+    incremental
+    column :id, :"varchar(32)", from: "Id"
+  end
+
+  table("proc_incremental_table") do
+    incremental if: -> { @proc_incremental }
+    column :id, :"varchar(32)", from: "Id"
+  end
+end
+
+TestDumper = InstDataShipper::Dumper.define(schema: TestSchema)

metadata

@@ -1,43 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: inst_data_shipper
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Instructure CustomDev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-20 00:00:00.000000000 Z
+date: 2024-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '6.0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -220,20 +206,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: apartment
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: with_model
   requirement: !ruby/object:Gem::Requirement
@@ -346,20 +318,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.18.0
-- !ruby/object:Gem::Dependency
-  name: rubyzip
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
@@ -405,6 +363,7 @@ files:
 - lib/inst_data_shipper/version.rb
 - spec/dummy/README.rdoc
 - spec/dummy/Rakefile
+- spec/dummy/app/models/application_record.rb
 - spec/dummy/bin/rails
 - spec/dummy/config.ru
 - spec/dummy/config/application.rb
@@ -419,9 +378,12 @@ files:
 - spec/dummy/config/routes.rb
 - spec/dummy/config/secrets.yml
 - spec/dummy/db/schema.rb
+- spec/inst_data_shipper/destinations/hosted_data_spec.rb
+- spec/inst_data_shipper/dumper_spec.rb
 - spec/spec_helper.rb
 - spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
 - spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv
+- spec/support/test_dumper.rb
 homepage: https://instructure.com
 licenses: []
 metadata: {}
@@ -440,27 +402,31 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: Gem for uploading app data to Instructure CustomDev Hosted Data tooling
 test_files:
-- spec/spec_helper.rb
-- spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv
-- spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
-- spec/dummy/db/schema.rb
+- spec/dummy/README.rdoc
 - spec/dummy/Rakefile
-- spec/dummy/config/secrets.yml
+- spec/dummy/app/models/application_record.rb
+- spec/dummy/bin/rails
 - spec/dummy/config/application.rb
-- spec/dummy/config/initializers/session_store.rb
-- spec/dummy/config/initializers/wrap_parameters.rb
-- spec/dummy/config/initializers/assets.rb
+- spec/dummy/config/boot.rb
+- spec/dummy/config/database.yml
+- spec/dummy/config/environment.rb
 - spec/dummy/config/environments/development.rb
 - spec/dummy/config/environments/test.rb
-- spec/dummy/config/database.yml
-- spec/dummy/config/boot.rb
+- spec/dummy/config/initializers/assets.rb
+- spec/dummy/config/initializers/session_store.rb
+- spec/dummy/config/initializers/wrap_parameters.rb
 - spec/dummy/config/routes.rb
-- spec/dummy/config/environment.rb
-- spec/dummy/README.rdoc
+- spec/dummy/config/secrets.yml
 - spec/dummy/config.ru
-- spec/dummy/bin/rails
+- spec/dummy/db/schema.rb
+- spec/inst_data_shipper/destinations/hosted_data_spec.rb
+- spec/inst_data_shipper/dumper_spec.rb
+- spec/spec_helper.rb
+- spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
+- spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv
+- spec/support/test_dumper.rb