redshift-connector-data_file 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f4a0fc069c6eb635d15751a45ccd2f2a7bf86c1
-  data.tar.gz: 4789569a3178cf601a307c349e3ba42e07830bb5
+  metadata.gz: 5fab0d487a44643d1add17ce0323edab71ac75b8
+  data.tar.gz: b415428c7d55384f2c7ac9b83ea537947247dfbd
 SHA512:
-  metadata.gz: 53e845941f6242c5804bcbc0918e8efe64a8cc825e896803ac2015b59774b199ec7ea43421aaf0a8ebd0cfc59cc394f663908174f2563fa34970d8159cc231e0
-  data.tar.gz: 434c36b9a6cfac9719e88245bdd187b1a6bd6ba28f9464ec611eefddfebfeb4b4e5549400ebc8fc08f0050470f31df3b9ada1f495f16c7962d414556e5cd15bb
+  metadata.gz: 8882dc7483274809e15bb4d6d1a7110aa5700946e300fb74399b3c895d7f0d1dd9a0aba7c61c6b5e0a5cf280251196efaa9022a89582111623898775a6910942
+  data.tar.gz: 0d5b894719f9ff572fcd9f4a6077adf7ed80ee6da8525a3dc77526a9403c243b32fb9b2337d450482f79b53fb81b5a37e44ae9a030829b4767ce10f5b72348bb

data/lib/redshift-connector/data_file/abstract_data_file_bundle.rb

@@ -18,5 +18,36 @@ module RedshiftConnector
     def all_data_objects
       data_files.select {|obj| obj.data_object? }
     end
+
+    REPORT_SIZE = 10_0000
+
+    def each_batch(report: true)
+      @logger.info "reader: #{@reader_class}"
+      n = 0
+      reported = 0
+      do_each_batch(@batch_size) do |rows|
+        yield rows
+        n += rows.size
+        if n / REPORT_SIZE > reported
+          @logger.info "#{n} rows processed" if report
+          reported = n / REPORT_SIZE
+        end
+      end
+      @logger.info "total #{n} rows processed" if report
+    end
+
+    def do_each_batch(batch_size)
+      filter = @filter
+      buf = []
+      each_row do |row|
+        buf.push filter.(*row)
+        if buf.size == batch_size
+          yield buf
+          buf = []
+        end
+      end
+      yield buf unless buf.empty?
+    end
+    private :do_each_batch
   end
 end

data/lib/redshift-connector/data_file/version.rb

@@ -1,5 +1,5 @@
 module RedshiftConnector
   module DataFile
-    VERSION = "1.0.0"
+    VERSION = "1.1.0"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redshift-connector-data_file
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Hidekazu Kobayashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-12 00:00:00.000000000 Z
+date: 2017-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler