norikra 1.1.2-java → 1.2.0-java

data/lib/norikra/server.rb

@@ -19,28 +19,28 @@ module Norikra
     attr_accessor :running
 
     MICRO_PREDEFINED = {
-      :engine => { inbound:    { threads: 0, capacity: 0 }, outbound:   { threads: 0, capacity: 0 },
-                   route_exec: { threads: 0, capacity: 0 }, timer_exec: { threads: 0, capacity: 0 }, },
-      :rpc => { threads: 2 }, # for desktop
-      :web => { threads: 2 },
+      engine: { inbound:    { threads: 0, capacity: 0 }, outbound:   { threads: 0, capacity: 0 },
+                route_exec: { threads: 0, capacity: 0 }, timer_exec: { threads: 0, capacity: 0 }, },
+      rpc: { threads: 2 }, # for desktop
+      web: { threads: 2 },
     }
     SMALL_PREDEFINED = {
-      :engine => { inbound:    { threads: 2, capacity: 0 }, outbound:   { threads: 2, capacity: 0 },
-                   route_exec: { threads: 2, capacity: 0 }, timer_exec: { threads: 2, capacity: 0 }, },
-      :rpc => { threads: 9 }, # 4core HT
-      :web => { threads: 9 },
+      engine: { inbound:    { threads: 2, capacity: 0 }, outbound:   { threads: 2, capacity: 0 },
+                route_exec: { threads: 2, capacity: 0 }, timer_exec: { threads: 2, capacity: 0 }, },
+      rpc: { threads: 9 }, # 4core HT
+      web: { threads: 9 },
     }
     MIDDLE_PREDEFINED = {
-      :engine => { inbound:    { threads: 4, capacity: 0 }, outbound:   { threads: 4, capacity: 0 },
-                   route_exec: { threads: 4, capacity: 0 }, timer_exec: { threads: 4, capacity: 0 }, },
-      :rpc => { threads: 17 }, # 4core HT 2CPU
-      :web => { threads: 17 },
+      engine: { inbound:    { threads: 4, capacity: 0 }, outbound:   { threads: 4, capacity: 0 },
+                route_exec: { threads: 4, capacity: 0 }, timer_exec: { threads: 4, capacity: 0 }, },
+      rpc: { threads: 17 }, # 4core HT 2CPU
+      web: { threads: 17 },
     }
     LARGE_PREDEFINED = {
-      :engine => { inbound:    { threads: 8, capacity: 0 }, outbound:   { threads: 8, capacity: 0 },
-                   route_exec: { threads: 8, capacity: 0 }, timer_exec: { threads: 8, capacity: 0 }, },
-      :rpc => { threads: 49 }, # 6core HT 4CPU
-      :web => { threads: 49 },
+      engine: { inbound:    { threads: 8, capacity: 0 }, outbound:   { threads: 8, capacity: 0 },
+                route_exec: { threads: 8, capacity: 0 }, timer_exec: { threads: 8, capacity: 0 }, },
+      rpc: { threads: 49 }, # 6core HT 4CPU
+      web: { threads: 49 },
     }
 
     def self.threading_configuration(conf)
@@ -118,14 +118,14 @@ module Norikra
       @engine = Norikra::Engine.new(@output_pool, @typedef_manager, {thread: @thread_conf[:engine]})
 
       @rpcserver = Norikra::RPC::HTTP.new(
-        :engine => @engine,
-        :host => @host, :port => @port,
-        :threads => @thread_conf[:rpc][:threads]
+        engine: @engine,
+        host: @host, port: @port,
+        threads: @thread_conf[:rpc][:threads]
       )
       @webserver = Norikra::WebUI::HTTP.new(
-        :engine => @engine,
-        :host => @host, :port => @ui_port,
-        :threads => @thread_conf[:web][:threads]
+        engine: @engine,
+        host: @host, port: @ui_port,
+        threads: @thread_conf[:web][:threads]
       )
     end
 
@@ -143,7 +143,7 @@ module Norikra
         end
         if @stats.queries && @stats.queries.size > 0
           @stats.queries.each do |query|
-            @engine.register(Norikra::Query.new(:name => query[:name], :group => query[:group], :expression => query[:expression]))
+            @engine.register(Norikra::Query.new(name: query[:name], group: query[:group], expression: query[:expression]))
           end
         end
       end
@@ -201,21 +201,27 @@ module Norikra
       info "Loading UDF plugins"
       Norikra::UDF.listup.each do |mojule|
         if mojule.is_a?(Class)
-          name = @engine.load(mojule)
-          info "UDF loaded", :name => name
+          name = @engine.load(:udf, mojule)
+          info "UDF loaded", name: name
         elsif mojule.is_a?(Module) && mojule.respond_to?(:plugins)
           mojule.init if mojule.respond_to?(:init)
           mojule.plugins.each do |klass|
-            name = @engine.load(klass)
-            info "UDF loaded", :name => name
+            name = @engine.load(:udf, klass)
+            info "UDF loaded", name: name
           end
         end
       end
+
+      info "Loading Listener plugins"
+      Norikra::Listener.listup.each do |klass|
+        @engine.load(:listener, klass)
+        info "Listener loaded", name: klass
+      end
     end
 
     def dump_stats
       Norikra::Stats.generate(@engine).dump(@stats_path, @stats_secondary_path)
-      info "Current status saved", :path => @stats_path
+      info "Current status saved", path: @stats_path
     end
   end
 end

data/lib/norikra/stats.rb

@@ -8,9 +8,9 @@ module Norikra
       Norikra::Stats.new(
         targets: engine.targets.map{|t|
           {
-            :name => t.name,
-            :fields => engine.typedef_manager.dump_target(t.name),
-            :auto_field => t.auto_field
+            name: t.name,
+            fields: engine.typedef_manager.dump_target(t.name),
+            auto_field: t.auto_field
           }
         },
         queries: engine.queries.map(&:dump)

data/lib/norikra/target.rb

@@ -18,7 +18,7 @@ module Norikra
     end
 
     def to_hash
-      {:name => @name, :auto_field => @auto_field}
+      {name: @name, auto_field: @auto_field}
     end
 
     def ==(other)

data/lib/norikra/typedef.rb

@@ -10,6 +10,7 @@ module Norikra
   #  * known field list of target (and these are optional or not), and container fields
   #  * known field-set list of a target
   #  * base set of a target
+
   class Typedef
     attr_accessor :fields, :container_fields, :waiting_fields ,:baseset, :queryfieldsets, :datafieldsets
 
@@ -39,7 +40,10 @@ module Norikra
       @queryfieldsets = []
       @datafieldsets = []
 
-      @set_map = {} # FieldSet.field_names_key(data_fieldset, fieldset) => data_fieldset
+      # FieldSet.field_names_key(data_fieldset, fieldset) => data_fieldset
+      ### field_names_key is built by keys w/ data, without null fields
+      ### data_fieldset includes null fields
+      @set_map = {}
 
       @mutex = Mutex.new
     end
@@ -94,7 +98,7 @@ module Norikra
 
     def push(level, fieldset)
       unless self.consistent?(fieldset)
-        warn "fieldset mismatch", :self => self, :with => fieldset
+        warn "fieldset mismatch", receiver: self, with: fieldset
         raise Norikra::ArgumentError, "field definition mismatch with already defined fields"
       end
 
@@ -131,7 +135,7 @@ module Norikra
             end
           end
         else
-          raise ArgumentError, "unknown level #{level}"
+          raise ::ArgumentError, "unknown level #{level}"
         end
       end
       true
@@ -147,7 +151,7 @@ module Norikra
         when :data
           raise RuntimeError, "BUG: pop of data fieldset is nonsense"
         else
-          raise ArgumentError, "unknown level #{level}"
+          raise ::ArgumentError, "unknown level #{level}"
         end
       end
       true
@@ -155,14 +159,14 @@ module Norikra
 
     def replace(level, old_fieldset, fieldset)
       unless self.consistent?(fieldset)
-        warn "fieldset mismatch", :self => self, :with => fieldset
+        warn "fieldset mismatch", receiver: self, with: fieldset
         raise Norikra::ArgumentError, "field definition mismatch with already defined fields"
       end
       if level != :data
-        raise ArgumentError, "invalid argument, fieldset replace should be called for :data"
+        raise ::ArgumentError, "invalid argument, fieldset replace should be called for :data"
       end
       if old_fieldset.field_names_key != fieldset.field_names_key
-        raise ArgumentError, "try to replace different field name sets"
+        raise ::ArgumentError, "try to replace different field name sets"
       end
       @mutex.synchronize do
         @datafieldsets.delete(old_fieldset)
@@ -173,9 +177,11 @@ module Norikra
     end
 
     def simple_guess(data, opts={})
-      unless opts.has_key?(:optional)
-        opts[:optional] = true
-      end
+      #### in typedef_manager
+      # guessed = @typedefs[target].simple_guess(event, strict: false, baseset: true)
+
+      #### in Typedef#refer
+      # guessed = self.simple_guess(data, strict: strict)
 
       flatten_key_value_pairs = []
 
@@ -199,24 +205,23 @@ module Norikra
       mapping = Hash[
         flatten_key_value_pairs.map{|key,value|
           type = case value
-                 when TrueClass,FalseClass then 'boolean'
-                 when Integer then 'long'
-                 when Float   then 'double'
-                 else
-                   'string'
+                 when TrueClass,FalseClass then {type: 'boolean'}
+                 when Integer then {type: 'long'}
+                 when Float   then {type: 'double'}
+                 else {type: 'string'}
                  end
           [key,type]
         }
       ]
 
-      FieldSet.new(mapping, opts[:optional])
+      FieldSet.new(mapping, false) # in simple_guess, optional is always false
     end
 
     def refer(data, strict=false)
       field_names_key = FieldSet.field_names_key(data, self, strict, @waiting_fields)
       return @set_map[field_names_key] if @set_map.has_key?(field_names_key)
 
-      guessed = self.simple_guess(data, optional: false, strict: strict)
+      guessed = self.simple_guess(data, strict: strict)
       guessed_fields = guessed.fields
       @fields.each do |key,field|
         if guessed_fields.has_key?(key)

data/lib/norikra/typedef_manager.rb

@@ -68,9 +68,11 @@ module Norikra
 
     def generate_fieldset_mapping(query)
       fields_set = {}
+      nullables_set = {}
 
       query.targets.each do |target|
         fields_set[target] = query.fields(target)
+        nullables_set[target] = query.nullable_fields(target)
       end
       query.fields(nil).each do |field|
         assumed = query.targets.select{|t| @typedefs[t].field_defined?([field])}
@@ -78,11 +80,12 @@ module Norikra
           raise Norikra::ClientError, "cannot determine target for field '#{field}' in this query"
         end
         fields_set[assumed.first].push(field)
+        nullables_set[assumed.first].push(field) if query.nullable_fields(nil).include?(field)
       end
 
       mapping = {}
       fields_set.each do |target,fields|
-        mapping[target] = generate_query_fieldset(target, fields.sort.uniq, query.name, query.group)
+        mapping[target] = generate_query_fieldset(target, fields.sort.uniq, nullables_set[target].sort.uniq, query.name, query.group)
       end
       mapping
     end
@@ -101,19 +104,19 @@ module Norikra
     end
 
     def generate_base_fieldset(target, event)
-      guessed = @typedefs[target].simple_guess(event, optional: false, strict: false, baseset: true) # all fields are non-optional
+      guessed = @typedefs[target].simple_guess(event, strict: false, baseset: true) # all fields are non-optional
       guessed.update(@typedefs[target].fields, false)
       guessed
     end
 
-    def generate_query_fieldset(target, field_name_list, query_name, query_group)
+    def generate_query_fieldset(target, field_name_list, nullable_list, query_name, query_group)
       # all fields of field_name_list should exists in definitions of typedef fields
       # for this premise, call 'bind_fieldset' for data fieldset before this method.
       required_fields = {}
       @mutex.synchronize do
         @typedefs[target].fields.each do |fieldname, field|
           if field_name_list.include?(fieldname) || !(field.optional?)
-            required_fields[fieldname] = {:type => field.type, :optional => field.optional}
+            required_fields[fieldname] = {type: field.type, optional: field.optional, nullable: nullable_list.include?(fieldname)}
           end
         end
       end
@@ -124,7 +127,7 @@ module Norikra
       @typedefs[target].baseset
     end
 
-    def subsets(target, fieldset) # for data fieldset
+    def subsets(target, fieldset) # manager.subsets(target, data_fieldset) #=> [query_fieldset]
       sets = []
       @mutex.synchronize do
         @typedefs[target].queryfieldsets.each do |set|
@@ -135,7 +138,7 @@ module Norikra
       sets
     end
 
-    def supersets(target, fieldset) # for query fieldset
+    def supersets(target, fieldset) # manager.supersets(target, query_fieldset) #=> [data_fieldset]
       sets = []
       @mutex.synchronize do
         @typedefs[target].datafieldsets.each do |set|

data/lib/norikra/udf.rb

@@ -1,6 +1,9 @@
 require 'norikra/error'
 require 'rubygems'
 
+require 'norikra/logger'
+include Norikra::Log
+
 module Norikra
   module UDF
     #### esper-4.9.0/esper/doc/reference/html/extension.html#custom-singlerow-function
@@ -97,17 +100,18 @@ module Norikra
       plugins = Gem.find_latest_files('norikra/udf/*.rb')
       plugins.each do |plugin|
         begin
-          debug "plugin file found!", :file => plugin
+          debug "plugin file found!", file: plugin
           rbpath = plugin.dup
           4.times do
             rbpath = File.dirname( rbpath )
           end
           files = Dir.entries( rbpath )
           gemname = files.select{|f| f=~ /\.gemspec$/ }.first.sub(/\.gemspec$/, '')
+          trace "Loading UDF gem", gemname: gemname, path: plugin
           require gemname
           load plugin
         rescue => e
-          warn "Failed to load norikra UDF plugin", :plugin => plugin.to_s, :error_class => e.class, :error => e.message
+          warn "Failed to load norikra UDF plugin", plugin: plugin.to_s, error_class: e.class, error: e.message
           e.backtrace.each do |t|
             warn "  " + t
           end

data/lib/norikra/version.rb

@@ -1,3 +1,3 @@
 module Norikra
-  VERSION = "1.1.2"
+  VERSION = "1.2.0"
 end

data/lib/norikra/webui/api.rb

@@ -30,9 +30,9 @@ class Norikra::WebUI::API < Sinatra::Base
 
   def logging(type, handler, args=[], opts={})
     if type == :manage
-      debug "WebAPI", :handler => handler.to_s, :args => args
+      debug "WebAPI", handler: handler.to_s, args: args
     else
-      trace "WebAPI", :handler => handler.to_s, :args => args
+      trace "WebAPI", handler: handler.to_s, args: args
     end
 
     begin
@@ -111,7 +111,7 @@ class Norikra::WebUI::API < Sinatra::Base
   post '/register' do
     query_name, query_group, expression = args = parse_args(['query_name', 'query_group', 'expression'], request)
     logging(:manage, :register, args){
-      r = engine.register(Norikra::Query.new(:name => query_name, :group => query_group, :expression => expression))
+      r = engine.register(Norikra::Query.new(name: query_name, group: query_group, expression: expression))
       json result: (!!r)
     }
   end

data/lib/norikra/webui/handler.rb

@@ -13,7 +13,7 @@ require 'erubis'
 class Norikra::WebUI::Handler < Sinatra::Base
   set :public_folder, File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'public'))
   set :views, File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'views'))
-  set :erb, :escape_html => true
+  set :erb, escape_html: true
 
   enable :sessions
 
@@ -27,9 +27,9 @@ class Norikra::WebUI::Handler < Sinatra::Base
 
   def logging(type, handler, args=[], opts={})
     if type == :manage
-      debug "WebUI", :handler => handler.to_s, :args => args
+      debug("WebUI"){ { handler: handler.to_s, args: args } }
     else
-      trace "WebUI", :handler => handler.to_s, :args => args
+      trace("WebUI"){ { handler: handler.to_s, args: args } }
     end
 
     begin
@@ -78,7 +78,7 @@ class Norikra::WebUI::Handler < Sinatra::Base
         }
       }
 
-      erb :index, :layout => :base, :locals => {
+      erb :index, layout: :base, locals: {
         input_data: input_data,
         stat: engine.statistics,
         queries: queries,
@@ -109,14 +109,14 @@ class Norikra::WebUI::Handler < Sinatra::Base
       redirect '/#query_add'
     }
 
-    logging(:manage, :register, [query_name, query_group, expression], :on_error_hook => error_hook) do
+    logging(:manage, :register, [query_name, query_group, expression], on_error_hook: error_hook) do
       if query_name.nil? || query_name.empty?
         raise Norikra::ClientError, "Query name MUST NOT be blank"
       end
       if query_group.nil? || query_group.empty?
         query_group = nil
       end
-      engine.register(Norikra::Query.new(:name => query_name, :group => query_group, :expression => expression))
+      engine.register(Norikra::Query.new(name: query_name, group: query_group, expression: expression))
       redirect '/#queries'
     end
   end

data/norikra.gemspec

@@ -31,6 +31,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
-  spec.add_development_dependency "spork"
   spec.add_development_dependency "pry"
 end

data/script/{spec_server_pry → pry}

@@ -1,27 +1,7 @@
 #!/usr/bin/env ruby
 
-require 'drb'
 require 'pry'
 
-begin
-  begin
-    DRb.start_service("druby://localhost:0")
-  rescue SocketError, Errno::EADDRNOTAVAIL
-    DRb.start_service("druby://:0")
-  end
-  $spec_server = DRbObject.new_with_uri("druby://127.0.0.1:8989")
-rescue DRb::DRbConnError
-  err.puts "No DRb server is running. Running in local process instead ..."
-end
-
-def rspec(file=nil)
-  if file
-    $spec_server.run(["--color", "--format", "s", file], STDERR, STDOUT)
-  else
-    $spec_server.run(["--color", 'spec'], STDERR, STDOUT)
-  end
-end
-
 $service = nil
 def service
   $service ||= com.espertech.esper.client.EPServiceProviderManager.getDefaultProvider
@@ -90,10 +70,6 @@ end
 puts <<DESC
 
 Example:
-  > # execute 'spork' on other terminal
-  > rspec 'spec/xxx_spec.rb'
-  > rspec # for all tests
-
   > query = Norikra::Query.new(name:'test1', expression:'SELECT ...')
   >
   > query.ast.to_a            # dump query AST

data/spec/field_spec.rb

@@ -135,6 +135,20 @@ describe Norikra::Field do
     end
   end
 
+  context 'specified as nullable' do
+    describe '#dup' do
+      it 'saves original boolean value' do
+        f = Norikra::Field.new('foo', 'string', false, false) # non-nullable
+        expect(f.nullable?).to be_falsy
+        expect(f.dup.nullable?).to be_falsy
+
+        f = Norikra::Field.new('bar', 'int', false, true)
+        expect(f.nullable?).to be_truthy
+        expect(f.dup.nullable?).to be_truthy
+      end
+    end
+  end
+
   context 'defined as string field' do
     describe '#format' do
       it 'converts specified value as string' do

data/spec/fieldset_spec.rb

@@ -36,11 +36,44 @@ describe Norikra::FieldSet do
       set = Norikra::FieldSet.new({'x' => 'string', 'y' => 'long', 'a' => 'Boolean'})
       expect(set.summary).to eql('a:boolean,x:string,y:integer')
     end
+
+    it 'can set optional/nullable options for each fields' do
+      set = Norikra::FieldSet.new({
+          'x' => {type: 'string'}, # optional: default_optional(==nil, falsy), nullable: false
+          'y' => {type: 'long', optional: true, nullable: true},
+        })
+      expect(set.fields['x'].type).to eql('string')
+      expect(set.fields['x'].optional?).to be_falsy
+      expect(set.fields['x'].nullable?).to be_falsy
+
+      expect(set.fields['y'].type).to eql('integer')
+      expect(set.fields['y'].optional?).to be_truthy
+      expect(set.fields['y'].nullable?).to be_truthy
+    end
+
+    it 'sets summary with nullable informations, ordered by key names' do
+      set = Norikra::FieldSet.new({
+          'x' => {type: 'string'}, # optional: default_optional(==nil, falsy), nullable: false
+          'y' => {type: 'integer', optional: true, nullable: true},
+          'z' => {type: 'boolean'}
+        })
+      expect(set.summary).to eql('x:string,y:integer:nullable,z:boolean')
+    end
   end
 
   context 'initialized with some fields' do
     set = Norikra::FieldSet.new({'x' => 'string', 'y' => 'long', 'a' => 'Boolean'})
     set2 = Norikra::FieldSet.new({'a' => 'string', 'b' => 'int', 'c' => 'float', 'd' => 'bool', 'e' => 'integer'})
+    set3 = Norikra::FieldSet.new({
+        'a' => 'string', 'b' => 'int', 'c' => 'float',
+        'd' => {type:'bool', optional: true, nullable: true},
+        'e' => {type:'long', optional: true, nullable: false}
+      })
+    set4 = Norikra::FieldSet.new({
+        'a' => 'string', 'b' => 'int', 'c' => 'float',
+        'd' => {type:'bool', optional: true, nullable: true},
+        'e' => {type:'long', optional: true, nullable: true}
+      })
 
     q_set1 = Norikra::FieldSet.new({'x' => 'string', 'y' => 'long'}, nil, 0, ['set1', 'g1'])
     q_set2 = Norikra::FieldSet.new({'x' => 'string', 'y' => 'long'}, nil, 0, ['set2', nil])
@@ -63,6 +96,15 @@ describe Norikra::FieldSet do
         expect(q_set2 == q_set2.dup).to be_truthy
         expect(q_set3 == q_set3.dup).to be_truthy
       end
+
+      it 'make duplicated object with same type/optional/nullable specifications' do
+        set3d = set3.dup
+        ['a', 'b', 'c', 'd', 'e'].each do |f|
+          expect(set3d.fields[f].type).to eql(set3.fields[f].type)
+          expect(set3d.fields[f].optional).to eql(set3.fields[f].optional)
+          expect(set3d.fields[f].nullable).to eql(set3.fields[f].nullable)
+        end
+      end
     end
 
     describe '.leaves' do
@@ -175,6 +217,11 @@ describe Norikra::FieldSet do
       it 'returns comma-separeted sorted field names' do
         expect(set.field_names_key).to eql('a,x,y')
       end
+
+      it 'returns result w/o nullable fields' do
+        expect(set3.field_names_key).to eql('a,b,c,e')
+        expect(set4.field_names_key).to eql('a,b,c')
+      end
     end
 
     describe '#udpate_summary' do
@@ -192,6 +239,18 @@ describe Norikra::FieldSet do
 
         expect(x.summary).not_to eql(oldsummary)
         expect(x.summary).to eql('a:boolean,x:integer,y:integer')
+
+        x.fields['c'] = Norikra::Field.new('c', 'string', true, true) # optional, nullable
+
+        x.update_summary
+
+        expect(x.summary).to eql('a:boolean,c:string:nullable,x:integer,y:integer')
+
+        x.fields['b'] = Norikra::Field.new('b', 'float', true, false) # optional, non-nullable
+
+        x.update_summary
+
+        expect(x.summary).to eql('a:boolean,b:float,c:string:nullable,x:integer,y:integer')
       end
     end
 
@@ -219,6 +278,31 @@ describe Norikra::FieldSet do
         x.update([Norikra::Field.new('z', 'string')], true)
         expect(x.fields.size).to eql(4)
         expect(x.summary).to eql('a:boolean,x:string,y:integer,z:string')
+        x.update([Norikra::Field.new('b', 'string', true, true), Norikra::Field.new('c', 'integer', true, true)], true)
+        expect(x.fields.size).to eql(6)
+        expect(x.summary).to eql('a:boolean,b:string:nullable,c:integer:nullable,x:string,y:integer,z:string')
+      end
+    end
+
+    describe '#nullable_diff' do
+      it 'provides the list of nullable fields, missing in self' do
+        q1 = Norikra::FieldSet.new({
+            'a' => 'string', 'd' => {type:'string', optional: false, nullable: true},
+          })
+        q2 = Norikra::FieldSet.new({
+            'a' => 'string', 'b' => 'int',
+            'd' => {type:'string', optional: false, nullable: true},
+            'e' => {type:'int', optional: false, nullable: true},
+          })
+
+        s1 = Norikra::FieldSet.new({'a' => 'string', 'b' => 'int', 'c' => 'bool'})
+        expect(s1.nullable_diff(q1).map(&:name).sort).to eql(['d'])
+
+        s2 = Norikra::FieldSet.new({'a' => 'string'})
+        expect(s2.nullable_diff(q1).map(&:name).sort).to eql(['d'])
+
+        s3 = Norikra::FieldSet.new({'a' => 'string', 'b' => 'int', 'c' => 'bool', 'd' => 'string'})
+        expect(s3.nullable_diff(q2).map(&:name).sort).to eql(['e'])
       end
     end
 
@@ -240,6 +324,15 @@ describe Norikra::FieldSet do
         expect(d['c']).to eql('double')
         expect(d['d']).to eql('boolean')
         expect(d['e']).to eql('long')
+
+        d = set3.definition # nullable does not have any effects
+        expect(d).to be_instance_of(Hash)
+        expect(d.size).to eql(5)
+        expect(d['a']).to eql('string')
+        expect(d['b']).to eql('long')
+        expect(d['c']).to eql('double')
+        expect(d['d']).to eql('boolean')
+        expect(d['e']).to eql('long')
       end
     end
 
@@ -254,6 +347,24 @@ describe Norikra::FieldSet do
         other.update([Norikra::Field.new('z', 'double')], false)
         expect(set.subset?(other)).to be_truthy
       end
+
+      it 'returns true if other instance does not have fields marked as nullable' do
+        other = Norikra::FieldSet.new({'a' => 'string', 'b' => 'int', 'c' => 'float', 'd' => 'bool', 'e' => 'int'})
+        expect(set3.subset?(other)).to be_truthy
+        expect(set4.subset?(other)).to be_truthy
+
+        other = Norikra::FieldSet.new({'a' => 'string', 'b' => 'int', 'c' => 'float'})
+        expect(set3.subset?(other)).to be_falsy
+        expect(set4.subset?(other)).to be_truthy
+
+        other = Norikra::FieldSet.new({'a' => 'string', 'b' => 'int', 'c' => 'float', 'd' => 'bool'})
+        expect(set3.subset?(other)).to be_falsy
+        expect(set4.subset?(other)).to be_truthy
+
+        other = Norikra::FieldSet.new({'a' => 'string', 'b' => 'int', 'c' => 'float', 'e' => 'int'})
+        expect(set3.subset?(other)).to be_truthy
+        expect(set4.subset?(other)).to be_truthy
+      end
     end
 
     describe '#bind' do