embulk-filter-ruby_proc 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 754bf37f61d14a24054997630406b8c9b02e7aff
-  data.tar.gz: c47c596b3745548ac8919e7be92727e6157bfd70
+SHA256:
+  metadata.gz: 727830dc138bf80017b10197c86129c82b61a1e0dc92a5c76548eaea8c23649f
+  data.tar.gz: 4cc54640d0b6f5906b4e91428553a1732adda5e8da4dd4bdfb5ee9a0dd712e8c
 SHA512:
-  metadata.gz: acf1ee4540152e40b7e706466b67f1f4f48eb23d200c988a3d46fe21dce669c19897c8f000cfbcf55f600c7c908a9accd7b51b268ecc7c9ad92f1cf1c5a733df
-  data.tar.gz: d3757d66ef78d46463a533b436fb4ae58c2e8a7fe09c9baad6c238eaca5204a5d6b76de095c7733ffa6e9f289056d7879e6e2e6fe534a82c7c83cbe55bde5fa6
+  metadata.gz: 460294e10ed0191f609d06236846c1a0acfcc70626f5b29ad45452ba84e6ddbab777747ded81594e787a63bc5eea9bd9b570d49a9fa906560bb325cd497c3152
+  data.tar.gz: 1cad7c00adb50981afc402c44ad60665aa41bae3ed0e1b6b2b5b492917b0334cb147aa0564f613666181ede81b49da90aec7efda74fc5de374f507216f36575d

data/README.md

@@ -51,6 +51,11 @@ filters:
           ->(record) do
             [record.dup, record.dup.tap { |r| r["id"] += 10 }]
           end
+    skip_rows:
+      - proc: |
+          ->(record) do
+            record["id"].odd?
+          end
     columns:
       - name: data
         proc: |
@@ -75,6 +80,8 @@ filters:
 
 ```
 
+If you want to skip record in "rows proc" or "columns proc", use `throw :skip_record`.
+
 ```rb
 # comment_upcase.rb
 
@@ -90,6 +97,16 @@ end
 - rows proc must return record hash or array of record hash.
   - user must take care of object identity. Otherwise, error may be occurred when plugin applys column procs.
 
+### proc execution order
+
+1. before procs
+1. per record
+    1. all row procs
+    1. per record applied row procs
+        1. all skip\_row procs
+        1. column procs
+1. after procs
+
 ### preview
 ```
 +-----------+--------------+-------------------------+-------------------------+------------------------------------------+------------------------------------------------------------------------------------------+

data/embulk-filter-ruby_proc.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-filter-ruby_proc"
-  spec.version       = "0.7.0"
+  spec.version       = "0.8.0"
   spec.authors       = ["joker1007"]
   spec.summary       = "Ruby Proc filter plugin for Embulk"
   spec.description   = "Filter each record by ruby proc"

data/lib/embulk/filter/ruby_proc.rb

@@ -33,6 +33,7 @@ module Embulk
         task = {
           "columns" => config.param("columns", :array, default: []),
           "rows" => config.param("rows", :array, default: []),
+          "skip_rows" => config.param("skip_rows", :array, default: []),
           "before" => config.param("before", :array, default: []),
           "after" => config.param("after", :array, default: []),
           "requires" => config.param("requires", :array, default: []),
@@ -55,6 +56,7 @@ module Embulk
 
         @proc_store ||= {}
         @row_proc_store ||= {}
+        @skip_row_proc_store ||= {}
         transaction_id = rand(100000000)
         until !@proc_store.has_key?(transaction_id)
           transaction_id = rand(100000000)
@@ -83,8 +85,15 @@ module Embulk
             eval(File.read(rowdef["proc_file"]), evaluator_binding, File.expand_path(rowdef["proc_file"]))
           end
         }.compact
+        @skip_row_proc_store[transaction_id] = skip_row_procs = task["skip_rows"].map {|rowdef|
+          if rowdef["proc"]
+            eval(rowdef["proc"], evaluator_binding)
+          else
+            eval(File.read(rowdef["proc_file"]), evaluator_binding, File.expand_path(rowdef["proc_file"]))
+          end
+        }.compact
         task["transaction_id"] = transaction_id
-        raise "Need columns or rows parameter" if procs.empty? && row_procs.empty?
+        raise "Need columns or rows parameter" if procs.empty? && row_procs.empty? && skip_row_procs.empty?
 
         before_procs.each do |pr|
           pr.call
@@ -113,6 +122,10 @@ module Embulk
         @row_proc_store
       end
 
+      def self.skip_row_proc_store
+        @skip_row_proc_store
+      end
+
       def self.parse_col_procs(columns, evaluator_binding)
         Hash[columns.map {|col|
           if col["proc"]
@@ -138,13 +151,15 @@ module Embulk
           require lib
         end
 
-        if self.class.proc_store.nil? || self.class.row_proc_store.nil?
+        if self.class.proc_store.nil? || self.class.row_proc_store.nil? || self.class.skip_row_proc_store.nil?
           evaluator_binding = Evaluator.new(task["variables"]).get_binding
           @procs = self.class.parse_col_procs(task["columns"], evaluator_binding)
           @row_procs = self.class.parse_row_procs(task["rows"], evaluator_binding)
+          @skip_row_procs = self.class.parse_row_procs(task["skip_rows"], evaluator_binding)
         else
           @procs = self.class.proc_store[task["transaction_id"]]
           @row_procs = self.class.row_proc_store[task["transaction_id"]]
+          @skip_row_procs = self.class.skip_row_proc_store[task["transaction_id"]]
         end
         @skip_nils = Hash[task["columns"].map {|col|
           [col["name"], col["skip_nil"].nil? ? true : !!col["skip_nil"]]
@@ -160,32 +175,40 @@ module Embulk
             record_hashes = [hashrize(record)]
           else
             record_hashes = row_procs.each_with_object([]) do |pr, arr|
-              result = pr.call(hashrize(record))
-              case result
-              when Array
-                result.each do |r|
-                  arr << r
+              catch :skip_record do
+                result = pr.call(hashrize(record))
+                case result
+                when Array
+                  result.each do |r|
+                    arr << r
+                  end
+                when Hash
+                  arr << result
+                else
+                  raise "row proc return value must be a Array or Hash"
                 end
-              when Hash
-                arr << result
-              else
-                raise "row proc return value must be a Array or Hash"
               end
             end
           end
 
           record_hashes.each do |record_hash|
-            procs.each do |col, pr|
-              next unless record_hash.has_key?(col)
-              next if record_hash[col].nil? && skip_nils[col]
-
-              if pr.arity == 1
-                record_hash[col] = pr.call(record_hash[col])
-              else
-                record_hash[col] = pr.call(record_hash[col], record_hash)
+            catch :skip_record do
+              skip_row_procs.each do |pr|
+                throw :skip_record if pr.call(record_hash)
               end
+
+              procs.each do |col, pr|
+                next unless record_hash.has_key?(col)
+                next if record_hash[col].nil? && skip_nils[col]
+
+                if pr.arity == 1
+                  record_hash[col] = pr.call(record_hash[col])
+                else
+                  record_hash[col] = pr.call(record_hash[col], record_hash)
+                end
+              end
+              page_builder.add(record_hash.values)
             end
-            page_builder.add(record_hash.values)
           end
         end
       end
@@ -208,6 +231,10 @@ module Embulk
         @row_procs
       end
 
+      def skip_row_procs
+        @skip_row_procs
+      end
+
       def skip_nils
         @skip_nils
       end

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-ruby_proc
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - joker1007
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-14 00:00:00.000000000 Z
+date: 2018-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: embulk
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.1
-  requirement: !ruby/object:Gem::Requirement
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.1
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
   name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
-  requirement: !ruby/object:Gem::Requirement
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
   name: rake
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-  requirement: !ruby/object:Gem::Requirement
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-  prerelease: false
-  type: :development
 description: Filter each record by ruby proc
 email:
 - kakyoin.hierophant@gmail.com
@@ -74,7 +74,7 @@ homepage: https://github.com/joker1007/embulk-filter-ruby_proc
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -89,9 +89,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.7.4
+signing_key: 
 specification_version: 4
 summary: Ruby Proc filter plugin for Embulk
 test_files: []