norikra 0.0.7-java → 0.0.8-java

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in norikra.gemspec
 gemspec
+
+gem "norikra-udf-woothee", :git => "file:///Users/tagomoris/Documents/norikra-udf-woothee"

data/README.md

@@ -86,6 +86,12 @@ And send more events:
 
 Query 'www.search' matches the last event automatically.
 
+## User Defined Functions
+
+UDF as plugins over rubygems are available. For example, see 'norikra-udf-mock' repository.
+
+TBD
+
 ## Versions
 
 TBD
@@ -94,7 +100,6 @@ TBD
 
 * daemonize
 * performance parameters
-* query unregister
 
 ## Copyright
 

data/lib/norikra/engine.rb

@@ -123,6 +123,10 @@ module Norikra
       nil
     end
 
+    def load(plugin_klass)
+      load_udf(plugin_klass)
+    end
+
     class Listener
       include com.espertech.esper.client.UpdateListener
 
@@ -282,6 +286,17 @@ module Norikra
       end
     end
 
+    def load_udf(udf_klass)
+      udf_klass.init if udf_klass.respond_to?(:init)
+
+      udf = udf_klass.new
+      if udf.is_a? Norikra::UDF::SingleRow
+        load_udf_actually(udf)
+      elsif udf.is_a? Norikra::UDF::AggregationSingle
+        load_udf_aggregation_actually(udf)
+      end
+    end
+
     # this method should be protected with @mutex lock
     def register_query_actually(query, mapping)
       # 'mapping' argument is {target => fieldset}
@@ -347,5 +362,32 @@ module Norikra
       debug "remove event type", :target => target, :event_type => event_type_name
       @config.removeEventType(event_type_name, true)
     end
+
+    VALUE_CACHE_ENUM = com.espertech.esper.client.ConfigurationPlugInSingleRowFunction::ValueCache
+    FILTER_OPTIMIZABLE_ENUM = com.espertech.esper.client.ConfigurationPlugInSingleRowFunction::FilterOptimizable
+
+    def load_udf_actually(udf)
+      debug "importing class into config object", :name => udf.class.to_s
+
+      functionName, className, methodName = udf.definition
+
+      valueCache = udf.value_cache ? VALUE_CACHE_ENUM::ENABLED : VALUE_CACHE_ENUM::DISABLED
+      filterOptimizable = udf.filter_optimizable ? FILTER_OPTIMIZABLE_ENUM::ENABLED : FILTER_OPTIMIZABLE_ENUM::DISABLED
+      rethrowExceptions = udf.rethrow_exceptions
+
+      debug "adding SingleRowFunction", :function => functionName, :javaClass => className, :javaMethod => methodName
+      @config.addPlugInSingleRowFunction(functionName, className, methodName, valueCache, filterOptimizable, rethrowExceptions)
+      functionName
+    end
+
+    def load_udf_aggregation_actually(udf)
+      debug "importing class into config object", :name => udf.class.to_s
+
+      functionName, factoryClassName = udf.definition
+
+      debug "adding AggregationSingleFactory", :function => functionName, :javaClass => factoryClassName
+      @config.addPlugInAggregationFunctionFactory(functionName, factoryClassName)
+      functionName
+    end
   end
 end

data/lib/norikra/rubyudf.rb

@@ -0,0 +1,49 @@
+# this is note for future update
+
+module Norikra
+  module UDF
+    class FailedUDFImplementationPureRuby
+      # require 'jruby/core_ext'
+      class WootheeIsCrawler < Norikra::UDF::Base # Norikra::UDF::WootheeIsCrawler < Norikra::UDF::Base
+        def self.init
+          require 'woothee'
+        end
+
+        def self.function_name
+          "isCrawler"
+        end
+
+        def self.isCrawler(agent)
+          Woothee.is_crawler(agent)
+        end
+        class << self
+          add_method_signature( "isCrawler", [java.lang.Boolean, java.lang.String] )
+        end
+      end
+
+      # for engine.rb
+      def load_udf_actually(udf_klass)
+        require 'jruby/core_ext'
+        udf_klass.init
+
+        jclass = udf_klass.become_java!(".")
+        className = jclass.get_name.to_java(:string)
+
+        #### try for NullPointerException, but doesn't work well
+        # field = jclass.getDeclaredField("ruby");
+        # field.setAccessible(java.lang.Boolean::TRUE)
+        # field.set(nil, org.jruby.Ruby.getGlobalRuntime)
+
+        functionName = udf_klass.function_name.to_java(:string)
+        methodName = udf_klass.method_name.to_java(:string)
+
+        valueCache = udf_klass.value_cache ? VALUE_CACHE_ENUM::ENABLED : VALUE_CACHE_ENUM::DISABLED
+        filterOptimizable = udf_klass.filter_optimizable ? FILTER_OPTIMIZABLE_ENUM::ENABLED : FILTER_OPTIMIZABLE_ENUM::DISABLED
+        rethrowExceptions = udf_klass.rethrow_exceptions
+
+        debug "adding SingleRowFunction", :class => udf_klass.to_s, :javaClass => jclass.get_name
+        @config.addPlugInSingleRowFunction(functionName, className, methodName, valueCache, filterOptimizable, rethrowExceptions)
+      end
+    end
+  end
+end

data/lib/norikra/server.rb

@@ -9,6 +9,7 @@ require 'norikra/typedef'
 require 'norikra/query'
 
 require 'norikra/rpc'
+require 'norikra/udf'
 
 module Norikra
   class Server
@@ -32,8 +33,11 @@ module Norikra
     def run
       @engine.start
       @rpcserver.start
-      info "Norikra server started."
+
+      load_plugins
+
       @running = true
+      info "Norikra server started."
 
       shutdown_proc = ->{ @running = false }
       # JVM uses SIGQUIT for thread/heap state dumping
@@ -47,6 +51,22 @@ module Norikra
       end
     end
 
+    def load_plugins
+      info "Loading UDF plugins"
+      Norikra::UDF.listup.each do |mojule|
+        if mojule.is_a?(Class)
+          name = @engine.load(mojule)
+          info "UDF loaded", :name => name
+        elsif mojule.is_a?(Module) && mojule.respond_to?(:plugins)
+          mojule.init if mojule.respond_to?(:init)
+          mojule.plugins.each do |klass|
+            name = @engine.load(klass)
+            info "UDF loaded", :name => name
+          end
+        end
+      end
+    end
+
     def shutdown
       info "Norikra server shutting down."
       @rpcserver.stop

data/lib/norikra/udf.rb

@@ -0,0 +1,124 @@
+require 'norikra/error'
+require 'rubygems'
+
+module Norikra
+  module UDF
+    #### esper-4.9.0/esper/doc/reference/html/extension.html#custom-singlerow-function
+    # <esper-configuration
+    #   <plugin-singlerow-function name="getDate"
+    #     function-class="mycompany.DateUtil" function-method="parseDate"
+    #     value-cache="enabled"
+    #     filter-optimizable="disabled"
+    #     rethrow-exceptions="disabled" />
+    # </esper-configuration>
+    class SingleRow
+      def self.init
+        true
+      end
+
+      def definition
+        [self.function_name, self.class_name, self.method_name]
+      end
+
+      # UDF function name in queries
+      def function_name
+        raise NotImplementedError
+      end
+
+      def class_name
+        raise NotImplementedError
+      end
+
+      def method_name
+        function_name
+      end
+
+      # 17.3.3. Value Cache
+      # When a single-row function receives parameters that are all constant values or expressions
+      # that themselves receive only constant values, Esper can pre-evaluate the result of
+      # the single-row function at time of statement creation.
+      # By default, Esper does not pre-evaluate the single-row function unless you configure
+      # the value cache as enabled.
+      def value_cache
+        false
+      end
+
+      # 17.3.4. Single-Row Functions in Filter Predicate Expressions
+      # Your EPL may use plug-in single row functions among the predicate expressions
+      # as part of the filters in a stream or pattern.
+      # For example, the EPL below uses the function computeHash as part of a predicate expression:
+      #
+      #    select * from MyEvent(computeHash(field) = 100)
+      #
+      # When you have many EPL statements or many context partitions that refer to the same function,
+      # event type and parameters in a predicate expression, the engine may optimize evaluation:
+      # The function gets evaluated only once per event.
+      #
+      # While the optimization is enabled by default for all plug-in single row functions,
+      # you can also disable the optimization for a specific single-row function.
+      # By disabling the optimization for a single-row function the engine may use less memory
+      # to identify reusable function footprints but may cause the engine to evaluate each function
+      # more frequently then necessary.
+      def filter_optimizable
+        true
+      end
+
+      # 17.3.7. Exception Handling
+      # By default the engine logs any exceptions thrown by the single row function and returns
+      # a null value. To have exceptions be re-thrown instead, which makes exceptions visible
+      # to any registered exception handler, please configure as discussed herein.
+      def rethrow_exceptions
+        false
+      end
+    end
+
+    class AggregationSingle
+      def self.init
+        true
+      end
+
+      def definition
+        [self.function_name, self.factory_class_name]
+      end
+
+      # UDF aggregation function name in queries
+      def function_name
+        raise NotImplementedError
+      end
+
+      def factory_class_name
+        raise NotImplementedError
+      end
+    end
+
+    def self.listup
+      return unless defined? Gem
+
+      #TODO: CHECK: same name files of different versions?
+      plugins = Gem.find_files('norikra/udf/*.rb')
+      plugins.each do |plugin|
+        begin
+          debug "plugin file found!", :file => plugin
+          load plugin
+        rescue => e
+          warn "Failed to load norikra UDF plugin", :plugin => plugin.to_s, :error_class => e.class, :error => e.message
+        end
+      end
+
+      known_consts = [:SingleRow, :AggregateSingle]
+      udf_bases = [Norikra::UDF::SingleRow, Norikra::UDF::AggregationSingle]
+      udfs = []
+      self.constants.each do |c|
+        next if known_consts.include?(c)
+
+        klass = Norikra::UDF.const_get(c)
+        if klass.is_a?(Class) && udf_bases.include?(klass.superclass)
+          udfs.push(klass)
+        elsif klass.is_a?(Module) && klass.respond_to?(:plugins)
+          udfs.push(klass)
+        end
+      end
+      udfs
+    end
+  end
+end

data/lib/norikra/udf_spec_helper.rb

@@ -0,0 +1,113 @@
+module Norikra
+  module UDFSpecHelper
+    @@plugins = {}
+
+    def function(name)
+      @@plugins[name.to_s]
+    end
+
+    def fcall(name, *args)
+      @@plugins[name.to_s]._call(*args)
+    end
+
+    # params: for AggregationSingle only
+    #         required keys: :valueType, :windowed, :distinct, :parameters
+    #           :parameters => [[parameterType, constant?, contantValue], ... ]
+    def udf_function(mojule, params={})
+      require 'esper-4.9.0.jar'
+
+      unless mojule.is_a?(Class)
+        mojule.init if mojule.respond_to?(:init)
+        ps = []
+        mojule.plugins.each{|p| ps.push(udf_function(p))} if mojule.respond_to?(:plugins)
+        return ps
+      end
+
+      klass = mojule
+      klass.init if klass.respond_to?(:init)
+
+      if klass.superclass == Norikra::UDF::SingleRow
+        name, classname, methodname = klass.new.definition
+        @@plugins[name] = UDFInstance.new(classname, methodname)
+
+      elsif klass.superclass == Norikra::UDF::AggregationSingle
+        name, factoryclassname = klass.new.definition
+        factory = UDFAggregationFactoryInstance.new(factoryclassname)
+        factory.check(name, params[:valueType], params[:parameters], params.fetch(:distinct, false), params.fetch(:windowed, true))
+        @@plugins[name] = factory.create
+      end
+    end
+
+    module UDFHelper
+      def classObject(classname)
+        parts = classname.split('.')
+        clazzname = parts.pop
+        eval("Java::" + parts.map(&:capitalize).join + "::" + clazzname)
+      end
+    end
+
+    class UDFInstance
+      include UDFHelper
+
+      def initialize(classname, methodname)
+        @methodname = methodname
+        @clazz = classObject(classname)
+      end
+
+      def _call(*args)
+        @clazz.send(@methodname.to_sym, *args)
+      end
+    end
+
+    class UDFAggregationInstance
+      def initialize(instance)
+        @func = instance
+      end
+
+      def _call(type, *args)
+        self.send(type, *args)
+      end
+
+      def getValueType; @func.getValueType; end # public Class getValueType()
+      def enter(*args); @func.enter(*args); end # public void enter(Object value)
+      def leave(*args); @func.leave(*args); end # public void leave(Object value)
+      def getValue; @func.getValue; end # public Object getValue()
+      def clear; @func.clear; end # public void clear()
+    end
+
+    class UDFAggregationFactoryInstance
+      include UDFHelper
+
+      def initialize(classname)
+        @factory = classObject(classname).new
+      end
+
+      # parameters => [[parameterType, constant?, contantValue], ... ]
+      def check(func_name, value_type, parameters, distinct, windowed)
+        @factory.setFunctionName(func_name)
+        unless @factory.getValueType == value_type.java_class
+          raise "Aggregation UDF value type mismatch, expected '#{value_type}', actually '#{@factory.getValueType}'"
+        end
+
+        parameterTypes = parameters.map{|t,bc,c| t.java_class }
+        constantValue  = parameters.map{|t,bc,c| bc.nil? ? false : bc }
+        constantValues = parameters.map{|t,bc,c| c }
+
+        # public AggregationValidationContext(java.lang.Class[] parameterTypes,
+        #                               boolean[] constantValue,
+        #                               java.lang.Object[] constantValues,
+        #                               boolean distinct,
+        #                               boolean windowed,
+        #                               ExprNode[] expressions)
+        context_class = com.espertech.esper.epl.agg.service.AggregationValidationContext
+        context = context_class.new(parameterTypes, constantValue, constantValues, distinct, windowed, [])
+        @factory.validate(context)
+        true
+      end
+
+      def create
+        UDFAggregationInstance.new(@factory.newAggregator)
+      end
+    end
+  end
+end

data/lib/norikra/version.rb

@@ -1,3 +1,3 @@
 module Norikra
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

metadata

@@ -2,14 +2,14 @@
 name: norikra
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 0.0.7
+  version: 0.0.8
 platform: java
 authors:
 - TAGOMORI Satoshi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-08-07 00:00:00.000000000 Z
+date: 2013-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mizuno
@@ -272,9 +272,12 @@ files:
 - lib/norikra/rpc/error.rb
 - lib/norikra/rpc/handler.rb
 - lib/norikra/rpc/http.rb
+- lib/norikra/rubyudf.rb
 - lib/norikra/server.rb
 - lib/norikra/typedef.rb
 - lib/norikra/typedef_manager.rb
+- lib/norikra/udf.rb
+- lib/norikra/udf_spec_helper.rb
 - lib/norikra/version.rb
 - norikra.gemspec
 - script/spec_server_pry