inst_data_shipper 0.2.8 → 0.2.10.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 75cf640fef52a5846408607d92b4cbeaedefec03e733542508c17ea245b7d3cf
-  data.tar.gz: 07bae82a7237af5d1720503a45f35208711c299c05485adc02f045de65462e49
+  metadata.gz: da87b45d1f05a92db75d71ce6fa414c4d30872bb2fe970f032151e0f0997378f
+  data.tar.gz: 25419ac15df71e1ec6e0d4b9dd0b3aca94ef5c812265386ce375cfb80c23d4f2
 SHA512:
-  metadata.gz: 81cb6a9801d399f0952f74083506938f9e3fd68b966a1afa4c24cbd9df393659ef01c1c4ae5cbd3a526809c2c1214966b798a9e30727d059fdfc00730a6931a1
-  data.tar.gz: 39930b880f8acd83230fda54bf559c9deb7fc95a31039212b7ab92eb81cc69b270cc0ed6e28237f0449a6e314146082d4c1f3203cc176eacf0b8c75dc6d3407f
+  metadata.gz: 20a9ffd8cf6934beb86d8e52048707263ebf77a8f43f071899649851d345aeecd789e7029baa0a31cefd6785cb0e0dc4c260f09dedaba6c83d09008805cd2235
+  data.tar.gz: b280466a01b6986bf418e978fb8be1edc7fd4d783d72173055d3deb78790296ffb6d04bd74416d2b0c6947f74d9defd9f6d8897b6b1e9b17fb6b1c6cb380bdd4

data/lib/inst_data_shipper/data_sources/base.rb

@@ -45,6 +45,13 @@ module InstDataShipper
           end
         end
       end
+
+      class MismatchingRowCounts < StandardError
+        def initialize(table_def={}, expected_row_count=0, processed_row_count=0)
+          message = "#{table_def[:warehouse_name]} expected #{expected_row_count} rows, processed #{processed_row_count} rows in local table."
+          super(message)
+        end
+      end
     end
   end
 end

data/lib/inst_data_shipper/data_sources/local_tables.rb

@@ -11,23 +11,46 @@ module InstDataShipper
 
         table_def = lookup_table_schema!(schema_name, { model: model })
 
-        inner_block = ->(file) {
-          query = model.all
-          query = _resolve_model_query(query, table_def[:query])
-
-          if table_is_incremental?(table_def)
-            query = _resolve_model_query(
-              query,
-              table_def.dig(:incremental, :scope),
-              string: ->(query, column) { query.where("#{column} > ?", incremental_since) },
-              default: "updated_at",
-            )
-          end
-
-          process_raw_data_enumerator(table_def, query.find_each, file)
+        upload_data(table_def, &make_data_factory(model, table_def))
+      end
+
+      def make_data_factory(model, table_def)
+        ->(file) {
+          query = build_query(model, table_def)
+          expected_row_count, processed_row_count = iterate_query(query, table_def, file)
+          raise MismatchingRowCounts.new(table_def, expected_row_count, processed_row_count) if !acceptable_row_counts_mismatch?(expected_row_count, processed_row_count)
         }
+      end
 
-        upload_data(table_def, &inner_block)
+      def build_query(model, table_def)
+        query = model.all
+        query = _resolve_model_query(query, table_def[:query])
+
+        if table_is_incremental?(table_def)
+          query = _resolve_model_query(
+            query,
+            table_def.dig(:incremental, :scope),
+            string: ->(query, column) { query.where("#{column} > ?", incremental_since) },
+            default: "updated_at",
+          )
+        end
+        query
+      end
+
+      def iterate_query(query, table_def, file)
+        processed_row_count = 0
+        expected_row_count = query.count
+
+        process_raw_data_enumerator(table_def, query.find_each.lazy.map { |x| processed_row_count += 1; x }, file)
+        [expected_row_count, processed_row_count]
+      end
+      #
+      # The count of rows in the database and the exported files may not always match. We accept a small range difference
+      def acceptable_row_counts_mismatch?(expected_row_count, processed_row_count, variation: 0.01)
+        return true if expected_row_count.nil?
+
+        expected_row_count = expected_row_count.to_f
+        expected_row_count * (1 - variation) <= processed_row_count && processed_row_count <= expected_row_count * (1 + variation)
       end
 
       private
@@ -47,7 +70,6 @@ module InstDataShipper
           raise "Invalid query: #{query.inspect}"
         end
       end
-
     end
   end
 end

data/lib/inst_data_shipper/dumper.rb

@@ -272,7 +272,6 @@ module InstDataShipper
           dest.upload_data_chunk(table_def, chunk)
         end
       end
-
       # TODO Consider how to handle errors in this method.
       #   Retriable errors must not be allowed to bubble - if dest 1 succeeds and dest 2 fails, dest 1 must not be retried
       #   Each destination should handle its own retries

data/lib/inst_data_shipper/version.rb

@@ -1,3 +1,3 @@
 module InstDataShipper
-  VERSION = "0.2.8".freeze
+  VERSION = "0.2.10.beta1".freeze
 end

data/spec/inst_data_shipper/data_sources/local_tables.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+RSpec.describe InstDataShipper::DataSources::LocalTables do
+  let(:dumper) do
+    dumper = TestDumper.new
+    dumper.extend(InstDataShipper::DataSources::LocalTables)
+    dumper
+  end
+
+  describe "#import_local_table" do
+    before do
+      allow(dumper).to receive(:iterate_query).and_return([5, 5])
+    end
+
+    it "counts expected rows and processes data" do
+      allow(dumper).to receive(:upload_data) do |td, &make_data_factory|
+        make_data_factory.call("filename")
+      end
+
+      expect { dumper.send(:_delayed_import_local_table, InstDataShipper::DumpBatch, schema_name: "local_table_test") }.not_to raise_error
+    end
+  end
+
+  describe '#make_data_factory' do
+    before do
+      allow(dumper).to receive(:build_query).and_return(true)
+    end
+    it 'raises an error when expected row count does not match the processed count' do
+      allow(dumper).to receive(:iterate_query).and_return([50, 100])
+      expect { dumper.make_data_factory(InstDataShipper::DumpBatch, {}).call("filename") }.to raise_error(InstDataShipper::DataSources::Base::MismatchingRowCounts)
+    end
+
+    it 'does not raise an error when the expected count matches the processed count' do
+      allow(dumper).to receive(:iterate_query).and_return([50, 50])
+      expect { dumper.make_data_factory(InstDataShipper::DumpBatch, {}).call("filename") }.not_to raise_error(InstDataShipper::DataSources::Base::MismatchingRowCounts)
+    end
+  end
+
+  describe '#acceptable_row_counts_mismatch?' do
+    it 'returns whether the counts are within the range or not' do
+      expect(dumper.acceptable_row_counts_mismatch?(100, 105, variation: 0.01)).to eq(false)
+      expect(dumper.acceptable_row_counts_mismatch?(100, 100, variation: 0.01)).to eq(true)
+      expect(dumper.acceptable_row_counts_mismatch?(100, 105, variation: 0.1)).to eq(true)
+      expect(dumper.acceptable_row_counts_mismatch?(100, 111, variation: 0.1)).to eq(false)
+    end
+
+    it 'returns false for the upper/lower bound' do
+      # current implementation accepts the upper / lower bound
+      expect(dumper.acceptable_row_counts_mismatch?(100, 101, variation: 0.01)).to eq(true)
+      expect(dumper.acceptable_row_counts_mismatch?(100, 99, variation: 0.01)).to eq(true)
+    end
+
+    it 'returns true for 0 rows expected and 0 rows processed' do
+      expect(dumper.acceptable_row_counts_mismatch?(0, 0, variation: 0.01)).to eq(true)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: inst_data_shipper
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.10.beta1
 platform: ruby
 authors:
 - Instructure CustomDev
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-11-04 00:00:00.000000000 Z
+date: 2026-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -361,6 +361,7 @@ files:
 - lib/inst_data_shipper/record.rb
 - lib/inst_data_shipper/schema_builder.rb
 - lib/inst_data_shipper/version.rb
+- spec/inst_data_shipper/data_sources/local_tables.rb
 - spec/inst_data_shipper/destinations/hosted_data_spec.rb
 - spec/inst_data_shipper/dumper_spec.rb
 - spec/internal/app/assets/config/manifest.js
@@ -396,6 +397,7 @@ signing_key:
 specification_version: 4
 summary: Gem for uploading app data to Instructure CustomDev Hosted Data tooling
 test_files:
+- spec/inst_data_shipper/data_sources/local_tables.rb
 - spec/inst_data_shipper/destinations/hosted_data_spec.rb
 - spec/inst_data_shipper/dumper_spec.rb
 - spec/internal/app/assets/config/manifest.js