embulk-filter-ruby_proc 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bdb10f169cbbee39d9e79d590ba36cd7bc43e739
-  data.tar.gz: 9d734ba55ecd29f87afcb340c968b6d066b308d1
+  metadata.gz: 7745c13afc58b5b1e5257cf7f1025fc3292b8103
+  data.tar.gz: fc8310448ff1cadba48dfb3cf59b64048749fc70
 SHA512:
-  metadata.gz: 2778a56657b6b19743ec75021c0a9dbeee32015dd10b315a85d7ac669acfef0218b045cb17c7e60303edb6c54d7737f2329acb936d2bed5e4651778f8e6e1d2d
-  data.tar.gz: 6f97934d1f9e6c6f656aa0739790f58258db028857b84f254007a0ba2e538bcc896c4ec8eec3229699971130d273501ee79f5747300d895373ee79e43bd9b1ad
+  metadata.gz: 9ca41e0fff89bb94f82f1c5035682ee48203ccae6e6493bd8867f6da3eaef1580618a15e442776ad21280d9cf96c8e8e5c5381a0ac1fbe811fb843702b38ca28
+  data.tar.gz: 8ba158fad7a3a5570bfdb3ad5186381cec5cf4a86eb927c022989e59a1e782fb5efa835ddc5d6470160b16edc28d1612402ad06ae2ce2be3019507b994422680

data/README.md

@@ -34,6 +34,18 @@ filters:
       - cgi
     variables:
       multiply: 3
+    before:
+      - proc: |
+          -> do
+            puts "before proc"
+            @started_at = Time.now
+          end
+    after:
+      - proc: |
+          -> do
+            puts "after proc"
+            p Time.now - @started_at
+          end
     rows:
       - proc: |
           ->(record) do
@@ -72,8 +84,11 @@ filters:
 end
 ```
 
-rows proc must return array of record hash.
-And user must take care of object identity. Otherwise, error may be occurred when plugin applys column procs.
+- `before` and `after` is executed at once
+- procs is evaluated on same binding (instance of Evaluator class)
+  - instance variable is shared
+- rows proc must return record hash or array of record hash.
+  - user must take care of object identity. Otherwise, error may be occurred when plugin applys column procs.
 
 ### preview
 ```

data/embulk-filter-ruby_proc.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-filter-ruby_proc"
-  spec.version       = "0.5.0"
+  spec.version       = "0.6.0"
   spec.authors       = ["joker1007"]
   spec.summary       = "Ruby Proc filter plugin for Embulk"
   spec.description   = "Filter each record by ruby proc"

data/example/config.yml

@@ -23,10 +23,20 @@ in:
 
 filters:
   - type: ruby_proc
-    requires:
-      - cgi
     variables:
       multiply: 3
+    before:
+      - proc: |
+          -> do
+            puts "before proc"
+            @started_at = Time.now
+          end
+    after:
+      - proc: |
+          -> do
+            puts "after proc"
+            p Time.now - @started_at
+          end
     rows:
       - proc: |
           ->(record) do
@@ -47,6 +57,11 @@ filters:
             id * variables["multiply"]
           end
         type: string
+
+  - type: ruby_proc
+    requires:
+      - cgi
+    columns:
       - name: comment
         proc_file: comment_upcase.rb
         skip_nil: false

data/lib/embulk/filter/ruby_proc.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'securerandom'
 
 module Embulk
   module Filter
@@ -32,6 +33,8 @@ module Embulk
         task = {
           "columns" => config.param("columns", :array, default: []),
           "rows" => config.param("rows", :array, default: []),
+          "before" => config.param("before", :array, default: []),
+          "after" => config.param("after", :array, default: []),
           "requires" => config.param("requires", :array, default: []),
           "variables" => config.param("variables", :hash, default: {}),
         }
@@ -50,37 +53,69 @@ module Embulk
           require lib
         end
 
+        @proc_store ||= {}
+        @row_proc_store ||= {}
+        transaction_id = rand(100000000)
+        until !@proc_store.has_key?(transaction_id)
+          transaction_id = rand(100000000)
+        end
         evaluator_binding = Evaluator.new(task["variables"]).get_binding
 
         # In order to avoid multithread probrem, initialize procs here
-        @procs = Hash[task["columns"].map {|col|
+        before_procs = task["before"].map {|before|
+          if before["proc"]
+            eval(before["proc"], evaluator_binding)
+          else
+            eval(File.read(before["proc_file"]), evaluator_binding, File.expand_path(before["proc_file"]))
+          end
+        }
+        @proc_store[transaction_id] = procs = Hash[task["columns"].map {|col|
           if col["proc"]
             [col["name"], eval(col["proc"], evaluator_binding)]
           else
             [col["name"], eval(File.read(col["proc_file"]), evaluator_binding, File.expand_path(col["proc_file"]))]
           end
         }]
-        @row_procs = task["rows"].map {|rowdef|
+        @row_proc_store[transaction_id] = row_procs = task["rows"].map {|rowdef|
           if rowdef["proc"]
             eval(rowdef["proc"], evaluator_binding)
           else
             eval(File.read(rowdef["proc_file"]), evaluator_binding, File.expand_path(rowdef["proc_file"]))
           end
         }.compact
-        raise "Need columns or rows parameter" if @row_procs.empty? && @procs.empty?
+        task["transaction_id"] = transaction_id
+        raise "Need columns or rows parameter" if procs.empty? && row_procs.empty?
+
+        before_procs.each do |pr|
+          pr.call
+        end
 
         yield(task, out_columns)
+
+        after_procs = task["after"].map {|after|
+          if after["proc"]
+            eval(after["proc"], evaluator_binding)
+          else
+            eval(File.read(after["proc_file"]), evaluator_binding, File.expand_path(after["proc_file"]))
+          end
+        }
+
+        after_procs.each do |pr|
+          pr.call
+        end
       end
 
-      def self.procs
-        @procs
+      def self.proc_store
+        @proc_store
       end
 
-      def self.row_procs
-        @row_procs
+      def self.row_proc_store
+        @row_proc_store
       end
 
       def init
+        @procs = self.class.proc_store[task["transaction_id"]]
+        @row_procs = self.class.row_proc_store[task["transaction_id"]]
         @skip_nils = Hash[task["columns"].map {|col|
           [col["name"], col["skip_nil"].nil? ? true : !!col["skip_nil"]]
         }]
@@ -136,11 +171,11 @@ module Embulk
       end
 
       def procs
-        self.class.procs
+        @procs
       end
 
       def row_procs
-        self.class.row_procs
+        @row_procs
       end
 
       def skip_nils

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-ruby_proc
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-14 00:00:00.000000000 Z
+date: 2016-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: embulk