embulk-filter-ruby_proc 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 727830dc138bf80017b10197c86129c82b61a1e0dc92a5c76548eaea8c23649f
-  data.tar.gz: 4cc54640d0b6f5906b4e91428553a1732adda5e8da4dd4bdfb5ee9a0dd712e8c
+  metadata.gz: 3ac1518db663e3846227546d746213c9c2f3f7a507ebce549c4137a75294ed54
+  data.tar.gz: a8cb61ed6b043444138bbe7e84fe0185f44377c27fc758b47b8227195f536740
 SHA512:
-  metadata.gz: 460294e10ed0191f609d06236846c1a0acfcc70626f5b29ad45452ba84e6ddbab777747ded81594e787a63bc5eea9bd9b570d49a9fa906560bb325cd497c3152
-  data.tar.gz: 1cad7c00adb50981afc402c44ad60665aa41bae3ed0e1b6b2b5b492917b0334cb147aa0564f613666181ede81b49da90aec7efda74fc5de374f507216f36575d
+  metadata.gz: f903a1672abada9756fb495e681ac517fdf0fbc1632b03b3c678ff0d5b4dc0ac02d776757c02e2fa035a2b1f322cd45b6a873d68ccfccbf0858d7b585e6fbbc8
+  data.tar.gz: e23dcfd2b09c1e88cc3cd73c12b65b2c362f96f547af245670640db8ced487255649b2e3fad93e98e697f5d729c3db6ff56ec1d6b29a8c4a87b501d00fb57813

data/.gitignore

@@ -4,3 +4,4 @@
 /.bundle/
 /Gemfile.lock
 /example/out*
+.ruby-version

data/README.md

@@ -75,6 +75,13 @@ filters:
         proc_file: comment_upcase.rb
         skip_nil: false
         type: json
+    pages:
+      - proc: |
+          ->(records) do
+            records.map do |record|
+              record.tap { |r| r["id"] += 1 }
+            end
+          end
 
 # ...
 
@@ -96,6 +103,8 @@ end
   - instance variable is shared
 - rows proc must return record hash or array of record hash.
   - user must take care of object identity. Otherwise, error may be occurred when plugin applys column procs.
+- pages proc must return array of record hash.
+  - use `page_size` option to increase size of processing record (ex. `-X page_size=64KB`)
 
 ### proc execution order
 
@@ -105,6 +114,7 @@ end
     1. per record applied row procs
         1. all skip\_row procs
         1. column procs
+1. per page procs
 1. after procs
 
 ### preview
@@ -113,13 +123,13 @@ end
 | id:string | account:long |          time:timestamp |      purchase:timestamp |                             comment:json |                                                                                data:json |
 +-----------+--------------+-------------------------+-------------------------+------------------------------------------+------------------------------------------------------------------------------------------+
 |         3 |       32,864 | 2015-01-27 19:23:49 UTC | 2015-01-27 00:00:00 UTC |                               ["EMBULK"] | {"events":[{"id":1,"name":"Name1","idx":0},{"id":2,"name":"Name2","idx":1}],"foo":"bar"} |
-|        33 |       32,864 | 2015-01-27 19:23:49 UTC | 2015-01-27 00:00:00 UTC |                               ["EMBULK"] | {"events":[{"id":1,"name":"Name1","idx":0},{"id":2,"name":"Name2","idx":1}],"foo":"bar"} |
+|        34 |       32,864 | 2015-01-27 19:23:49 UTC | 2015-01-27 00:00:00 UTC |                               ["EMBULK"] | {"events":[{"id":1,"name":"Name1","idx":0},{"id":2,"name":"Name2","idx":1}],"foo":"bar"} |
 |         6 |       14,824 | 2015-01-27 19:01:23 UTC | 2015-01-27 00:00:00 UTC |                       ["EMBULK","JRUBY"] |                                                                                          |
-|        36 |       14,824 | 2015-01-27 19:01:23 UTC | 2015-01-27 00:00:00 UTC |                       ["EMBULK","JRUBY"] |                                                                                          |
+|        37 |       14,824 | 2015-01-27 19:01:23 UTC | 2015-01-27 00:00:00 UTC |                       ["EMBULK","JRUBY"] |                                                                                          |
 |         9 |       27,559 | 2015-01-28 02:20:02 UTC | 2015-01-28 00:00:00 UTC | ["EMBULK","%22CSV%22","PARSER","PLUGIN"] |                                                                                          |
-|        39 |       27,559 | 2015-01-28 02:20:02 UTC | 2015-01-28 00:00:00 UTC | ["EMBULK","%22CSV%22","PARSER","PLUGIN"] |                                                                                          |
+|        40 |       27,559 | 2015-01-28 02:20:02 UTC | 2015-01-28 00:00:00 UTC | ["EMBULK","%22CSV%22","PARSER","PLUGIN"] |                                                                                          |
 |        12 |       11,270 | 2015-01-29 11:54:36 UTC | 2015-01-29 00:00:00 UTC |                                ["11270"] |                                                                                          |
-|        42 |       11,270 | 2015-01-29 11:54:36 UTC | 2015-01-29 00:00:00 UTC |                                ["11270"] |                                                                                          |
+|        43 |       11,270 | 2015-01-29 11:54:36 UTC | 2015-01-29 00:00:00 UTC |                                ["11270"] |                                                                                          |
 +-----------+--------------+-------------------------+-------------------------+------------------------------------------+------------------------------------------------------------------------------------------+
 ```
 

data/embulk-filter-ruby_proc.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-filter-ruby_proc"
-  spec.version       = "0.8.0"
+  spec.version       = "0.8.1"
   spec.authors       = ["joker1007"]
   spec.summary       = "Ruby Proc filter plugin for Embulk"
   spec.description   = "Filter each record by ruby proc"

data/example/config.yml

@@ -57,6 +57,13 @@ filters:
             id * variables["multiply"]
           end
         type: string
+    pages:
+      - proc: |
+          ->(records) do
+            records.map do |record|
+              record.tap { |r| r["id"] += 1 }
+            end
+          end
 
   - type: ruby_proc
     requires:

data/lib/embulk/filter/ruby_proc.rb

@@ -33,6 +33,7 @@ module Embulk
         task = {
           "columns" => config.param("columns", :array, default: []),
           "rows" => config.param("rows", :array, default: []),
+          "pages" => config.param("pages", :array, default: []),
           "skip_rows" => config.param("skip_rows", :array, default: []),
           "before" => config.param("before", :array, default: []),
           "after" => config.param("after", :array, default: []),
@@ -56,6 +57,7 @@ module Embulk
 
         @proc_store ||= {}
         @row_proc_store ||= {}
+        @page_proc_store ||= {}
         @skip_row_proc_store ||= {}
         transaction_id = rand(100000000)
         until !@proc_store.has_key?(transaction_id)
@@ -85,6 +87,13 @@ module Embulk
             eval(File.read(rowdef["proc_file"]), evaluator_binding, File.expand_path(rowdef["proc_file"]))
           end
         }.compact
+        @page_proc_store[transaction_id] = page_procs = task["pages"].map {|page|
+          if page["proc"]
+            eval(page["proc"], evaluator_binding)
+          else
+            eval(File.read(page["proc_file"]), evaluator_binding, File.expand_path(page["proc_file"]))
+          end
+        }.compact
         @skip_row_proc_store[transaction_id] = skip_row_procs = task["skip_rows"].map {|rowdef|
           if rowdef["proc"]
             eval(rowdef["proc"], evaluator_binding)
@@ -93,7 +102,9 @@ module Embulk
           end
         }.compact
         task["transaction_id"] = transaction_id
-        raise "Need columns or rows parameter" if procs.empty? && row_procs.empty? && skip_row_procs.empty?
+        if procs.empty? && row_procs.empty? && page_procs.empty? && skip_row_procs.empty?
+          raise "Need columns or rows or pages parameter"
+        end
 
         before_procs.each do |pr|
           pr.call
@@ -122,6 +133,10 @@ module Embulk
         @row_proc_store
       end
 
+      def self.page_proc_store
+        @page_proc_store
+      end
+
       def self.skip_row_proc_store
         @skip_row_proc_store
       end
@@ -146,19 +161,31 @@ module Embulk
         }.compact
       end
 
+      def self.parse_page_procs(pages, evaluator_binding)
+        pages.map {|page|
+          if page["proc"]
+            eval(page["proc"], evaluator_binding)
+          else
+            eval(File.read(page["proc_file"]), evaluator_binding, File.expand_path(page["proc_file"]))
+          end
+        }.compact
+      end
+
       def init
         task["requires"].each do |lib|
           require lib
         end
 
-        if self.class.proc_store.nil? || self.class.row_proc_store.nil? || self.class.skip_row_proc_store.nil?
+        if self.class.proc_store.nil? || self.class.row_proc_store.nil? || self.class.page_proc_store.nil? || self.class.skip_row_proc_store.nil?
           evaluator_binding = Evaluator.new(task["variables"]).get_binding
           @procs = self.class.parse_col_procs(task["columns"], evaluator_binding)
           @row_procs = self.class.parse_row_procs(task["rows"], evaluator_binding)
+          @page_procs = self.class.parse_page_procs(task["pages"], evaluator_binding)
           @skip_row_procs = self.class.parse_row_procs(task["skip_rows"], evaluator_binding)
         else
           @procs = self.class.proc_store[task["transaction_id"]]
           @row_procs = self.class.row_proc_store[task["transaction_id"]]
+          @page_procs = self.class.page_proc_store[task["transaction_id"]]
           @skip_row_procs = self.class.skip_row_proc_store[task["transaction_id"]]
         end
         @skip_nils = Hash[task["columns"].map {|col|
@@ -170,6 +197,7 @@ module Embulk
       end
 
       def add(page)
+        proc_records = []
         page.each do |record|
           if row_procs.empty?
             record_hashes = [hashrize(record)]
@@ -207,10 +235,21 @@ module Embulk
                   record_hash[col] = pr.call(record_hash[col], record_hash)
                 end
               end
-              page_builder.add(record_hash.values)
+              if page_procs.empty?
+                page_builder.add(record_hash.values)
+              else
+                proc_records << record_hash
+              end
             end
           end
         end
+
+        unless page_procs.empty?
+          page_procs.each do |pr|
+            result = pr.call(proc_records)
+            result.each { |record| page_builder.add(record.values) }
+          end
+        end
       end
 
       def finish
@@ -231,6 +270,10 @@ module Embulk
         @row_procs
       end
 
+      def page_procs
+        @page_procs
+      end
+
       def skip_row_procs
         @skip_row_procs
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-ruby_proc
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-28 00:00:00.000000000 Z
+date: 2019-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: embulk
@@ -60,7 +60,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".ruby-version"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -89,8 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Ruby Proc filter plugin for Embulk

data/.ruby-version

@@ -1 +0,0 @@
-jruby-9.0.4.0