tapsoob 0.7.10 → 0.7.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45c3ae3f925966118c644e975b4450342adab5cad2fa95b89ed9c30c55250be3
-  data.tar.gz: 661fe1a397453f233fc959ff2dbfd79ba208de8ef47d1588154a905b51980954
+  metadata.gz: bb1f0ec12a5b2c4add9d789368f7a46d143e4065b7828d2674735b913009e15a
+  data.tar.gz: 89ab50fb4d02fac6c6efc403498d82e8f895bc7de48b2bd8130d4ff1a2756023
 SHA512:
-  metadata.gz: e3eb503dab2d13a0620a6c97c9687e5fcdf6a2f8c90d4ed1920ded03c02a512a379ac036430416115d2b3b14db969f43bb2715bd780b49370f301068f6be36e5
-  data.tar.gz: 5b80973c77976e74456d059d42ca41f2c36a24320e6f31d03f25a40d9e3f02fc2bcd4b6d7507f9502e49d57133925ebd6015ba5c7e39c4b110cce7470cfb110f
+  metadata.gz: 87dacb466fb4041ae0ac10062145fc24e59f70d3f3036b7f7e7fe3a671c7eb33f646201583ba243f73153ff06febecbfd1a9316af01d318518164f2bb32c7baa
+  data.tar.gz: 742e89d4c8222594265f1f811a52935a125efd948bd1d356b86ae848ae59c6a18917c3d7ef95d999bd9116647d64ad30dc0023951c4cddc9157c81fdb8f22c19

data/lib/tapsoob/data_stream/interleaved.rb

@@ -17,23 +17,26 @@ module Tapsoob
         }.merge(@state)
       end
 
-      def fetch_rows
+      def next_offset
         worker_id = state[:worker_id]
         num_workers = state[:num_workers]
         chunk_number = state[:chunk_number]
         chunksize = state[:chunksize]
+        global_chunk_index = (chunk_number * num_workers) + worker_id
+        global_chunk_index * chunksize
+      end
+
+      def fetch_rows
+        worker_id = state[:worker_id]
+        num_workers = state[:num_workers]
+        chunksize = state[:chunksize]
 
-        # Only count once on first fetch
         state[:size] ||= table.count
 
-        # Calculate which global chunk this worker should fetch
-        # Worker 0: chunks 0, num_workers, 2*num_workers, ...
-        # Worker 1: chunks 1, num_workers+1, 2*num_workers+1, ...
-        global_chunk_index = (chunk_number * num_workers) + worker_id
-        offset = global_chunk_index * chunksize
+        offset = next_offset
 
         ds = table.order(*order_by).limit(chunksize, offset)
-        log.debug "DataStream::Interleaved#fetch_rows SQL -> #{ds.sql} (worker #{worker_id}/#{num_workers}, chunk #{chunk_number})"
+        log.debug "DataStream::Interleaved#fetch_rows SQL -> #{ds.sql} (worker #{worker_id}/#{num_workers}, chunk #{state[:chunk_number]})"
 
         rows = Tapsoob::Utils.format_data(db, ds.all,
           :string_columns => string_columns,
@@ -74,7 +77,7 @@ module Tapsoob
 
       def complete?
         state[:size] ||= table.count
-        state[:offset] >= state[:size]
+        state[:offset] >= state[:size] || next_offset >= state[:size]
       end
     end
   end

data/lib/tapsoob/operation/base.rb

@@ -126,8 +126,14 @@ module Tapsoob
         opts[:stream_state] = val
       end
 
+      def max_intra_table_workers
+        available_cpus = Etc.nprocessors rescue 4
+        [available_cpus / 2, 8, 2].max
+      end
+
       def db
-        @db ||= Sequel.connect(database_url, max_connections: parallel_workers * 2)
+        pool_size = parallel_workers * max_intra_table_workers + 2
+        @db ||= Sequel.connect(database_url, max_connections: pool_size)
         @db.extension :schema_dumper
         @db.loggers << Tapsoob.log if opts[:debug]
 

data/lib/tapsoob/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Tapsoob
-  VERSION = "0.7.10".freeze
+  VERSION = "0.7.12".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tapsoob
 version: !ruby/object:Gem::Version
-  version: 0.7.10
+  version: 0.7.12
 platform: ruby
 authors:
 - Félix Bellanger