umbrellio-utils 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d17fd4116bc29709f96f27f812fdf8f9b5e977a1f0982ec6ca39616c6458e879
-  data.tar.gz: 64de97d755d6759be25ff38cc07d1680d4501f355982e22003f59520c21c01bb
+  metadata.gz: 4ed0221575b9cfc5c5350ea8eed168a30e9804b943c20fc357c8ee36c963d767
+  data.tar.gz: 18eb1859efe000d5c5d2bb13b57217a84651f0f824542f49c8093439b49d8141
 SHA512:
-  metadata.gz: f32e5a5768428d0a2b7d3a8b0737916947088c35490ea8da10bddad4296683eac86612494cabae0dff9a50b6ec4aeb29dcd2adba7677133079282ae4fea7d7a0
-  data.tar.gz: 559fba254624cee4c16bb6f078cff3ea7c3b40f64579429d663fb89ff4c90e6afff20ec9f8d76cbb94c134598ab833c09cc5a74d61553cfd45e757145268553d
+  metadata.gz: 2e4077e807ee85ebc7e4db39f3f3e94ee5b75e94fde085082aadcd3d5422a0827522f00d37022cbb1bf6df71a051d5a1faaad53468870fa189f3f496e17d5531
+  data.tar.gz: cdd6bce77b7ab07d9ebc975c08551c73f0804540ae9fbff1145965118db40ec0a7f0719b25166ea6e4e5bff2bbc2d01a316b20cfd4a9761074fa9e93bd9d7987

data/Gemfile

@@ -5,25 +5,24 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in umbrellio_utils.gemspec
 gemspec
 
-group :development, :test do
-  gem "activesupport"
-  gem "bundler"
-  gem "ci-helper"
-  gem "http"
-  gem "nokogiri"
-  gem "nori"
-  gem "pg"
-  gem "pry"
-  gem "rake"
-  gem "rspec"
-  gem "rspec-json_matcher"
-  gem "rubocop-config-umbrellio"
-  gem "semantic_logger"
-  gem "sequel"
-  gem "sequel-batches"
-  gem "simplecov"
-  gem "simplecov-lcov"
-  gem "table_sync"
-  gem "timecop"
-  gem "yard"
-end
+gem "activesupport"
+gem "bundler"
+gem "ci-helper"
+gem "http"
+gem "net-pop", github: "ruby/net-pop" # See https://stackoverflow.com/questions/78617432/strange-bundle-update-issue-disappearing-net-pop-0-1-2-dependency
+gem "nokogiri"
+gem "nori"
+gem "pg"
+gem "pry"
+gem "rake"
+gem "rspec"
+gem "rspec-json_matcher"
+gem "rubocop-config-umbrellio"
+gem "semantic_logger"
+gem "sequel"
+gem "sequel-batches"
+gem "simplecov"
+gem "simplecov-lcov"
+gem "table_sync"
+gem "timecop"
+gem "yard"

data/Gemfile.lock

@@ -1,7 +1,14 @@
+GIT
+  remote: https://github.com/ruby/net-pop.git
+  revision: e8d0afe2773b9eb6a23c39e9e437f6fc0fc7c733
+  specs:
+    net-pop (0.1.2)
+      net-protocol
+
 PATH
   remote: .
   specs:
-    umbrellio-utils (1.3.0)
+    umbrellio-utils (1.4.0)
       memery (~> 1)
 
 GEM
@@ -106,7 +113,7 @@ GEM
     docile (1.4.0)
     domain_name (0.6.20240107)
     drb (2.2.1)
-    dry-inflector (1.0.0)
+    dry-inflector (1.1.0)
     erubi (1.13.0)
     ffi (1.17.0)
     ffi (1.17.0-arm64-darwin)
@@ -158,7 +165,6 @@ GEM
     net-imap (0.4.14)
       date
       net-protocol
-    net-pop (0.1.2)
     net-protocol (0.2.2)
       timeout
     net-smtp (0.5.0)
@@ -193,7 +199,7 @@ GEM
       sneakers (~> 2.0)
       tainbox
     racc (1.8.0)
-    rack (3.1.4)
+    rack (3.1.6)
     rack-session (2.0.0)
       rack (>= 3.0.0)
     rack-test (2.1.0)
@@ -304,7 +310,7 @@ GEM
     self_data (1.3.0)
     semantic_logger (4.15.0)
       concurrent-ruby (~> 1.0)
-    sequel (5.81.0)
+    sequel (5.82.0)
       bigdecimal
     sequel-batches (2.0.2)
       sequel
@@ -367,6 +373,7 @@ DEPENDENCIES
   bundler
   ci-helper
   http
+  net-pop!
   nokogiri
   nori
   pg

data/lib/umbrellio_utils/database.rb

@@ -22,8 +22,8 @@ module UmbrellioUtils
       error.result.error_field(PG::Result::PG_DIAG_CONSTRAINT_NAME)
     end
 
-    def each_record(dataset, **options, &block)
-      primary_key = primary_key_from(dataset, **options)
+    def each_record(dataset, primary_key: nil, **options, &block)
+      primary_key = primary_key_from(dataset, primary_key: primary_key)
 
       with_temp_table(dataset, **options) do |ids|
         rows = ids.map { |id| row(id.is_a?(Hash) ? id.values : [id]) }
@@ -31,11 +31,28 @@ module UmbrellioUtils
       end
     end
 
-    def with_temp_table(dataset, page_size: 1_000, sleep: nil, **options)
-      primary_key = primary_key_from(dataset, **options)
+    # Iterates over a dataset and yields batches of primary keys.
+    # First, a temporary table is created and populated with dataset primary keys.
+    # After that, a batch of rows is deleted from the temp table on each iteration
+    # and gets yielded to the caller.
+    # @option [Integer] page_size max size of each yielded PK batch
+    # @option [Integer] sleep interval to sleep between each iteration
+    # @option [Array] primary_key custom primary key to use for dataset
+    # @option [Symbol, String] temp_table_name custom name for temporary table,
+    #   table is reused if already exists
+    def with_temp_table(
+      dataset,
+      page_size: 1_000,
+      sleep: nil,
+      primary_key: nil,
+      temp_table_name: nil
+    )
+      primary_key = primary_key_from(dataset, primary_key: primary_key)
       sleep_interval = sleep_interval_from(sleep)
 
-      temp_table_name = create_temp_table(dataset, primary_key: primary_key)
+      temp_table_name = create_temp_table(
+        dataset, primary_key: primary_key, temp_table_name: temp_table_name&.to_sym
+      )
 
       pk_set = []
 
@@ -49,23 +66,26 @@ module UmbrellioUtils
 
         Kernel.sleep(sleep_interval) if sleep_interval.positive?
       end
-    ensure
+
       DB.drop_table(temp_table_name)
     end
 
-    def create_temp_table(dataset, **options)
+    def create_temp_table(dataset, primary_key: nil, temp_table_name: nil)
       time = Time.current
       model = dataset.model
-      temp_table_name = :"temp_#{model.table_name}_#{time.to_i}_#{time.nsec}"
-      primary_key = primary_key_from(dataset, **options)
+
+      temp_table_name ||= :"temp_#{model.table_name}_#{time.to_i}_#{time.nsec}"
+      return temp_table_name if DB.table_exists?(temp_table_name)
+
+      primary_key = primary_key_from(dataset, primary_key: primary_key)
 
       DB.create_table(temp_table_name, unlogged: true) do
         primary_key.each do |field|
           type = model.db_schema[field][:db_type]
-          column field, type
+          column(field, type)
         end
 
-        primary_key primary_key
+        primary_key(primary_key)
       end
 
       insert_ds = dataset.select(*qualified_pk(model.table_name, primary_key))

data/lib/umbrellio_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module UmbrellioUtils
-  VERSION = "1.3.0"
+  VERSION = "1.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: umbrellio-utils
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Team Umbrellio
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-07-01 00:00:00.000000000 Z
+date: 2024-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: memery