norikra 0.0.13-java → 0.0.14-java

data/lib/norikra/query/ast.rb

@@ -52,6 +52,33 @@ module Norikra
     #      "double"],
     #     "10.5"]]]]
 
+    ### SELECT count(*) AS cnt
+    ### FROM TestTable.win:time_batch(10 sec)
+    ### WHERE params.$$path.$1="/" AND size.$0.bytes > 100 and opts.num.seq.length() > 0
+
+    # ["EPL_EXPR",
+    #  ["SELECTION_EXPR", ["SELECTION_ELEMENT_EXPR", "count", "cnt"]],
+    #  ["STREAM_EXPR",
+    #   ["EVENT_FILTER_EXPR", "TestTable"],
+    #   ["VIEW_EXPR", "win", "time_batch", ["TIME_PERIOD", ["SECOND_PART", "10"]]]],
+    #  ["WHERE_EXPR",
+    #   ["EVAL_AND_EXPR",
+    #    ["EVAL_EQUALS_EXPR",
+    #     ["EVENT_PROP_EXPR",
+    #      ["EVENT_PROP_SIMPLE", "params"],
+    #      ["EVENT_PROP_SIMPLE", "$$path"],
+    #      ["EVENT_PROP_SIMPLE", "$1"]],
+    #     "\"/\""],
+    #    [">",
+    #     ["EVENT_PROP_EXPR",
+    #      ["EVENT_PROP_SIMPLE", "size"],
+    #      ["EVENT_PROP_SIMPLE", "$0"],
+    #      ["EVENT_PROP_SIMPLE", "bytes"]],
+    #     "100"],
+    #    [">",
+    #     ["LIB_FUNC_CHAIN", ["LIB_FUNCTION", "opts.num.seq", "length", "("]],
+    #     "0"]]]]
+
     def astnode(tree)
       children = if tree.children
                    tree.children.map{|c| astnode(c)}
@@ -114,23 +141,28 @@ module Norikra
         result
       end
 
-      def fields(default_target=nil)
-        @children.map{|c| c.nodetype?(:subquery) ? [] : c.fields(default_target)}.reduce(&:+) || []
+      def fields(default_target=nil, known_targets_aliases=[])
+        @children.map{|c| c.nodetype?(:subquery) ? [] : c.fields(default_target, known_targets_aliases)}.reduce(&:+) || []
       end
     end
 
     class ASTEventPropNode < ASTNode # EVENT_PROP_EXPR
-      # ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "bbb"]]
-      # ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "fraud"], ["EVENT_PROP_SIMPLE", "aaa"]]
+      # "bbb"           => ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "bbb"]]
+      # "fraud.aaa"     => ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "fraud"], ["EVENT_PROP_SIMPLE", "aaa"]]
+      # "size.$0.bytes" => ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "size"],  ["EVENT_PROP_SIMPLE", "$0"], ["EVENT_PROP_SIMPLE", "bytes"]]
 
       def nodetype?(*sym)
         sym.include?(:prop) || sym.include?(:property)
       end
 
-      def fields(default_target=nil)
+      def fields(default_target=nil, known_targets_aliases=[])
         props = self.listup('EVENT_PROP_SIMPLE')
-        if props.size > 1 # alias.fieldname
-          [ {:f => props[1].child.name, :t => props[0].child.name} ]
+        if props.size > 1 # alias.fieldname or container_fieldname.key.$1
+          if known_targets_aliases.include?(props[0].child.name)
+            [ {:f => props[1..-1].map{|n| n.child.name}.join("."), :t => props[0].child.name} ]
+          else
+            [ {:f => props.map{|n| n.child.name}.join("."), :t => default_target} ]
+          end
         else # fieldname (default target)
           [ {:f => props[0].child.name, :t => default_target } ]
         end
@@ -138,42 +170,53 @@ module Norikra
     end
 
     class ASTLibFunctionNode < ASTNode # LIB_FUNCTION
-      # ["LIB_FUNCTION", "now", "("]                 #### now()
-      # ["LIB_FUNCTION", "hoge", "length", "("] #    #### hoge.length()
-      # ["LIB_FUNCTION", "hoge", "substr", "0", "("] #### hoge.substr(0)
-      # ["LIB_FUNCTION", "substr", "10", "0", "("]   #### substr(10,0)
-      # ["LIB_FUNCTION", "hoge", "substr", "0", "8", "("] #### hoge.substr(0,8)
-      # ["LIB_FUNCTION", "max", ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "size"]], "("] #### max(size)
+      # "now()"            => ["LIB_FUNCTION", "now", "("]
+      # "hoge.length()"    => ["LIB_FUNCTION", "hoge", "length", "("]
+      # "hoge.substr(0)"   => ["LIB_FUNCTION", "hoge", "substr", "0", "("]
+      # "substr(10,0)"     => ["LIB_FUNCTION", "substr", "10", "0", "("]
+      # "hoge.substr(0,8)" => ["LIB_FUNCTION", "hoge", "substr", "0", "8", "("]
+
+      # "opts.num.$0.length()" => ["LIB_FUNCTION", "opts.num.$0", "length", "("]
+
+      # "max(size)"             => ["LIB_FUNCTION", "max", ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "size"]], "("]
 
       def nodetype?(*sym)
         sym.include?(:lib) || sym.include?(:libfunc)
       end
 
-      def fields(default_target=nil)
+      def fields(default_target=nil, known_targets_aliases=[])
         if @children.size <= 2
           # single function like 'now()', function-name and "("
           []
 
         elsif @children[1].nodetype?(:prop, :lib, :subquery)
           # first element should be func name if second element is property, library call or subqueries
-          self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target)}.reduce(&:+) || []
+          self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target, known_targets_aliases)}.reduce(&:+) || []
 
         elsif @children[1].name =~ /^(-)?\d+(\.\d+)$/ || @children[1].name =~ /^'[^']*'$/ || @children[1].name =~ /^"[^"]*"$/
           # first element should be func name if secod element is number/string literal
-          self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target)}.reduce(&:+) || []
+          self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target, known_targets_aliases)}.reduce(&:+) || []
 
         elsif Norikra::Query.imported_java_class?(@children[0].name)
           # Java imported class name (ex: 'Math.abs(-1)')
-          self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target)}.reduce(&:+) || []
+          self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target, known_targets_aliases)}.reduce(&:+) || []
 
         else
-          # first element may be property (simple 'fieldname.funcname()' or fully qualified 'target.fieldname.funcname()')
+          # first element may be property
+          #  * simple 'fieldname.funcname()'
+          #  * fully qualified 'target.fieldname.funcname()'
+          #  * simple/fully-qualified container field access 'fieldname.key.$0.funcname()' or 'target.fieldname.$1.funcname()'
           target,fieldname = if @children[0].name.include?('.')
-                               @children[0].name.split('.', 2)
+                               parts = @children[0].name.split('.')
+                               if known_targets_aliases.include?(parts[0])
+                                 [ parts[0], parts[1..-1].join(".") ]
+                               else
+                                 [ default_target, @children[0].name ]
+                               end
                              else
                                [default_target,@children[0].name]
                              end
-          children_list = self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target)}.reduce(&:+) || []
+          children_list = self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target, known_targets_aliases)}.reduce(&:+) || []
           [{:f => fieldname, :t => target}] + children_list
         end
       end
@@ -202,9 +245,9 @@ module Norikra
         @children.last.children.size < 1 ? @children.last.name : nil
       end
 
-      def fields(default_target=nil)
+      def fields(default_target=nil, known_targets_aliases=[])
         this_target = self.target
-        self.listup('EVENT_PROP_EXPR').map{|p| p.fields(this_target)}.reduce(&:+) || []
+        self.listup('EVENT_PROP_EXPR').map{|p| p.fields(this_target,known_targets_aliases)}.reduce(&:+) || []
       end
     end
 

data/lib/norikra/rpc/handler.rb

@@ -32,13 +32,13 @@ class Norikra::RPC::Handler
 
   def targets
     logging(:show, :targets){
-      @engine.targets
+      @engine.targets.map(&:to_hash)
     }
   end
 
-  def open(target, fields)
+  def open(target, fields, auto_field)
     logging(:manage, :open, target, fields){
-      r = @engine.open(target, fields)
+      r = @engine.open(target, fields, auto_field)
       !!r
     }
   end
@@ -50,6 +50,13 @@ class Norikra::RPC::Handler
     }
   end
 
+  def modify(target, auto_field)
+    logging(:manage, :modify, target, auto_field){
+      r = @engine.modify(target, auto_field)
+      !!r
+    }
+  end
+
   def queries
     logging(:show, :queries){
       @engine.queries.map(&:to_hash)

data/lib/norikra/server.rb

@@ -108,7 +108,7 @@ module Norikra
         info "loading from stats file"
         if @stats.targets && @stats.targets.size > 0
           @stats.targets.each do |target|
-            @engine.open(target[:name], target[:fields])
+            @engine.open(target[:name], target[:fields], target[:auto_field])
           end
         end
         if @stats.queries && @stats.queries.size > 0
@@ -163,7 +163,13 @@ module Norikra
           port: @port,
           threads: @thread_conf,
           log: @log_conf,
-          targets: @engine.typedef_manager.dump,
+          targets: @engine.targets.map{|t|
+            {
+              :name => t.name,
+              :fields => @engine.typedef_manager.dump_target(t.name),
+              :auto_field => t.auto_field
+            }
+          },
           queries: @engine.queries.map{|q| {:name => q.name, :expression => q.expression}}
         )
         stats.dump(@stats_path)

data/lib/norikra/target.rb

@@ -1,7 +1,31 @@
 module Norikra
   class Target
+    attr_accessor :name, :fields, :auto_field
+
     def self.valid?(target_name)
       target_name =~ /^[a-zA-Z]([_a-zA-Z0-9]*[a-zA-Z0-9])?$/
     end
+
+    def initialize(name, fields=[], auto_field=true)
+      @name = name
+      @fields = fields
+      @auto_field = !!auto_field
+    end
+
+    def to_s
+      @name
+    end
+
+    def to_hash
+      {:name => @name, :auto_field => @auto_field}
+    end
+
+    def ==(other)
+      self.class == other.class ? self.name == other.name : self.name == other.to_s
+    end
+
+    def auto_field?
+      @auto_field
+    end
   end
 end

data/lib/norikra/typedef.rb

@@ -1,238 +1,41 @@
-require 'digest'
 require 'json'
 
 require 'norikra/error'
 
-# Norikra::Field, Norikra::FieldSet, Norikra::Typedef
+require 'norikra/field'
+require 'norikra/fieldset'
 
 module Norikra
-  class Field
-    attr_accessor :name, :type, :optional
-
-    def initialize(name, type, optional=nil)
-      @name = name.to_s
-      @type = self.class.valid_type?(type)
-      @optional = optional
-    end
-
-    def to_hash(sym=false)
-      if sym
-        {name: @name, type: @type, optional: @optional}
-      else
-        {'name' => @name, 'type' => @type, 'optional' => @optional}
-      end
-    end
-
-    def dup(optional=nil)
-      self.class.new(@name, @type, optional.nil? ? @optional : optional)
-    end
-
-    def ==(other)
-      self.name == other.name && self.type == other.type && self.optional == other.optional
-    end
-
-    def optional? # used outside of FieldSet
-      @optional
-    end
-
-    ### esper types
-    ### http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html/epl_clauses.html#epl-syntax-datatype
-    # string  A single character to an unlimited number of characters.
-    # boolean A boolean value.
-    # integer An integer value (4 byte).
-    # long    A long value (8 byte). Use the "L" or "l" (lowercase L) suffix. # select 1L as field1, 1l as field2
-    # double  A double-precision 64-bit IEEE 754 floating point.              # select 1.67 as field1, 167e-2 as field2, 1.67d as field3
-    # float   A single-precision 32-bit IEEE 754 floating point. Use the "f" suffix. # select 1.2f as field1, 1.2F as field2
-    # byte    A 8-bit signed two's complement integer.                        # select 0x10 as field1
-    #
-    #### 'integer' in epser document IS WRONG.
-    #### If 'integer' specified, esper raises this exception:
-    ### Exception: Nestable type configuration encountered an unexpected property type name 'integer' for property 'status',
-    ### expected java.lang.Class or java.util.Map or the name of a previously-declared Map or ObjectArray type
-    #### Correct type name is 'int'. see and run 'junks/esper-test.rb'
-    def self.valid_type?(type)
-      case type.to_s.downcase
-      when 'string' then 'string'
-      when 'boolean' then 'boolean'
-      when 'int' then 'int'
-      when 'long' then 'long'
-      when 'float' then 'float'
-      when 'double' then 'double'
-      else
-        raise Norikra::ArgumentError, "invalid field type '#{type}'"
-      end
-    end
-
-    def format(value)
-      case @type
-      when 'string'  then value.to_s
-      when 'boolean' then value =~ /^(true|false)$/i ? ($1.downcase == 'true') : (!!value)
-      when 'long','int' then value.to_i
-      when 'double','float' then value.to_f
-      else
-        raise RuntimeError, "unknown field type (in format), maybe BUG. name:#{@name},type:#{@type}"
-      end
-    end
-  end
-
-  class FieldSet
-    attr_accessor :summary, :fields
-    attr_accessor :target, :level
-
-    def initialize(fields, default_optional=nil, rebounds=0)
-      @fields = {}
-      fields.keys.each do |key|
-        data = fields[key]
-        type,optional = if data.is_a?(Hash)
-                          [data[:type].to_s, (data.has_key?(:optional) ? data[:optional] : default_optional)]
-                        elsif data.is_a?(String) || data.is_a?(Symbol)
-                          [data.to_s, default_optional]
-                        else
-                          raise ArgumentError, "FieldSet.new argument class unknown: #{fields.class}"
-                        end
-        @fields[key.to_s] = Field.new(key.to_s, type, optional)
-      end
-      self.update_summary
-
-      @target = nil
-      @level = nil
-      @rebounds = rebounds
-      @event_type_name = nil
-    end
-
-    def dup
-      fields = Hash[@fields.map{|key,field| [key, {:type => field.type, :optional => field.optional}]}]
-      self.class.new(fields, nil, @rebounds)
-    end
-
-    def self.field_names_key(data, fieldset=nil)
-      if fieldset
-        keys = []
-        fieldset.fields.each do |key,field|
-          unless field.optional?
-            keys.push(key)
-          end
-        end
-        data.keys.each do |key|
-          keys.push(key) unless keys.include?(key)
-        end
-        keys.sort.join(',')
-      else
-        data.keys.sort.join(',')
-      end
-    end
-
-    def field_names_key
-      self.class.field_names_key(@fields)
-    end
-
-    def update_summary
-      @summary = @fields.keys.sort.map{|k| @fields[k].name + ':' + @fields[k].type}.join(',')
-      self
-    end
-
-    def update(fields, optional_flag)
-      fields.each do |field|
-        @fields[field.name] = field.dup(optional_flag)
-      end
-      self.update_summary
-    end
-
-    #TODO: have a bug?
-    def ==(other)
-      return false if self.class != other.class
-      self.summary == other.summary
-    end
-
-    def definition
-      d = {}
-      @fields.each do |key, field|
-        d[field.name] = field.type
-      end
-      d
-    end
-
-    def subset?(other) # self is subset of other (or not)
-      (self.fields.keys - other.fields.keys).size == 0
-    end
-
-    def event_type_name
-      @event_type_name.dup
-    end
-
-    def bind(target, level, update_type_name=false)
-      @target = target
-      @level = level
-      prefix = case level
-               when :base then 'b_'
-               when :query then 'q_'
-               when :data then 'e_' # event
-               else
-                 raise ArgumentError, "unknown fieldset bind level: #{level}, for target #{target}"
-               end
-      @rebounds += 1 if update_type_name
-
-      @event_type_name = prefix + Digest::MD5.hexdigest([target, level.to_s, @rebounds.to_s, @summary].join("\t"))
-      self
-    end
-
-    def rebind(update_type_name)
-      self.dup.bind(@target, @level, update_type_name)
-    end
-
-    def self.simple_guess(data, optional=true)
-      mapping = Hash[
-        data.map{|key,value|
-          type = case value
-                 when TrueClass,FalseClass then 'boolean'
-                 when Integer then 'long'
-                 when Float   then 'double'
-                 else
-                   'string'
-                 end
-          [key,type]
-        }
-      ]
-      self.new(mapping, optional)
-    end
-
-    # def self.guess(data, optional=true)
-    #   mapping = Hash[
-    #     data.map{|key,value|
-    #       sval = value.to_s
-    #       type = case
-    #              when val.is_a?(TrueClass) || val.is_a?(FalseClass) || sval =~ /^(?:true|false)$/i
-    #                'boolean'
-    #              when val.is_a?(Integer) || sval =~ /^-?\d+[lL]?$/
-    #                'long'
-    #              when val.is_a?(Float) || sval =~ /^-?\d+\.\d+(?:[eE]-?\d+|[dDfF])?$/
-    #                'double'
-    #              else
-    #                'string'
-    #              end
-    #       [key,type]
-    #     }
-    #   ]
-    #   self.new(mapping, optional)
-    # end
-  end
-
   # Typedef is
-  #  * known field list of target (and these are optional or not)
+  #  * known field list of target (and these are optional or not), and container fields
   #  * known field-set list of a target
   #  * base set of a target
   class Typedef
-    attr_accessor :fields, :baseset, :queryfieldsets, :datafieldsets
+    attr_accessor :fields, :container_fields, :waiting_fields ,:baseset, :queryfieldsets, :datafieldsets
 
     def initialize(fields=nil)
+      @container_fields = {}
+
       if fields && !fields.empty?
-        @baseset = FieldSet.new(fields, false) # all fields are required
-        @fields = @baseset.fields.dup
+        given_set = FieldSet.new(fields, false) # default_optional=false, but chained access fields are always optional
+        @fields = given_set.fields.dup
+        base_fields = {}
+        @fields.values.each do |field|
+          if !field.optional?
+            base_fields[field.name] = field
+          elsif field.chained_access?
+            cname = field.container_name
+            @container_fields[cname] = Norikra::Field.new(cname, field.container_type, true)
+          end
+        end
+        @baseset = FieldSet.new(base_fields, false)
       else
         @baseset = nil
         @fields = {}
       end
 
+      @waiting_fields = []
+
       @queryfieldsets = []
       @datafieldsets = []
 
@@ -242,6 +45,8 @@ module Norikra
     end
 
     def field_defined?(list)
+      # used for only queries: TypedefManager#ready?(query) and TypedefManager#generate_field_mapping(query)
+      #  be not needed to think about containers
       list.reduce(true){|r,f| r && @fields[f]}
     end
 
@@ -256,6 +61,9 @@ module Norikra
           set.fields[fieldname] = field.dup(false)
         end
         @baseset = set
+        @baseset.fields.each do |name,f|
+          @waiting_fields.delete(name) if @waiting_fields.include?(name)
+        end
         @fields = @baseset.fields.merge(@fields)
       end
     end
@@ -264,7 +72,15 @@ module Norikra
       fieldname = fieldname.to_s
       @mutex.synchronize do
         return false if @fields[fieldname]
-        @fields[fieldname] = Field.new(fieldname, type, optional)
+        field = Norikra::Field.new(fieldname, type, optional)
+        if @waiting_fields.include?(fieldname)
+          @waiting_fields.delete(fieldname)
+        end
+        @fields[fieldname] = field
+        if field.chained_access? && !@container_fields[field.container_name]
+          container = Norikra::Field.new(field.container_name, field.container_type, true)
+          @container_fields[field.container_name] = container
+        end
       end
       true
     end
@@ -272,12 +88,13 @@ module Norikra
     def consistent?(fieldset)
       fields = fieldset.fields
       @baseset.subset?(fieldset) &&
-        @fields.values.select{|f| !f.optional? }.reduce(true){|r,f| r && fields[f.name] && fields[f.name].type == f.type} &&
-        fields.values.reduce(true){|r,f| r && (@fields[f.name].nil? || @fields[f.name].type == f.type)}
+        @fields.values.select{|f| !f.optional? }.reduce(true){|r,f| r && fields.has_key?(f.name) && fields[f.name].type == f.type} &&
+        fields.values.reduce(true){|r,f| r && (!@fields.has_key?(f.name) || @fields[f.name].type == f.type)}
     end
 
     def push(level, fieldset)
       unless self.consistent?(fieldset)
+        warn "fieldset mismatch", :self => self, :with => fieldset
         raise Norikra::ArgumentError, "field definition mismatch with already defined fields"
       end
 
@@ -292,6 +109,7 @@ module Norikra
             @queryfieldsets.push(fieldset)
 
             fieldset.fields.each do |fieldname,field|
+              @waiting_fields.delete(fieldname) if @waiting_fields.include?(fieldname)
               @fields[fieldname] = field.dup(true) unless @fields[fieldname]
             end
           end
@@ -301,7 +119,15 @@ module Norikra
             @set_map[fieldset.field_names_key] = fieldset
 
             fieldset.fields.each do |fieldname,field|
-              @fields[fieldname] = field.dup(true) unless @fields[fieldname]
+              if @waiting_fields.include?(fieldname)
+                @waiting_fields.delete(fieldname)
+              end
+              unless @fields[fieldname]
+                @fields[fieldname] = field.dup(true)
+                if field.chained_access? && !@container_fields[field.container_name]
+                  @container_fields[field.container_name] = Norikra::Field.new(field.container_name, field.container_type, true)
+                end
+              end
             end
           end
         else
@@ -329,6 +155,7 @@ module Norikra
 
     def replace(level, old_fieldset, fieldset)
       unless self.consistent?(fieldset)
+        warn "fieldset mismatch", :self => self, :with => fieldset
         raise Norikra::ArgumentError, "field definition mismatch with already defined fields"
       end
       if level != :data
@@ -345,11 +172,65 @@ module Norikra
       true
     end
 
-    def refer(data)
-      field_names_key = FieldSet.field_names_key(data, self)
+    def simple_guess(data, optional=true, strict=false)
+      flatten_key_value_pairs = []
+
+      data.each do |key,value|
+        next if strict && !(@fields.has_key?(key) || @waiting_fields.include?(key) || value.is_a?(Hash) || value.is_a?(Array))
+
+        if value.is_a?(Hash) || value.is_a?(Array)
+          Norikra::FieldSet.leaves(value).map{|chain| [key] + chain}.each do |chain|
+            value = chain.pop
+            key = Norikra::Field.regulate_key_chain(chain).join('.')
+            next unless @fields.has_key?(key) || @waiting_fields.include?(key)
+            flatten_key_value_pairs.push([key, value])
+          end
+        else
+          flatten_key_value_pairs.push([key, value])
+        end
+      end
+
+      mapping = Hash[
+        flatten_key_value_pairs.map{|key,value|
+          type = case value
+                 when TrueClass,FalseClass then 'boolean'
+                 when Integer then 'long'
+                 when Float   then 'double'
+                 else
+                   'string'
+                 end
+          [key,type]
+        }
+      ]
+
+      FieldSet.new(mapping, optional)
+    end
+
+    # def self.guess(data, optional=true)
+    #   mapping = Hash[
+    #     data.map{|key,value|
+    #       sval = value.to_s
+    #       type = case
+    #              when val.is_a?(TrueClass) || val.is_a?(FalseClass) || sval =~ /^(?:true|false)$/i
+    #                'boolean'
+    #              when val.is_a?(Integer) || sval =~ /^-?\d+[lL]?$/
+    #                'long'
+    #              when val.is_a?(Float) || sval =~ /^-?\d+\.\d+(?:[eE]-?\d+|[dDfF])?$/
+    #                'double'
+    #              else
+    #                'string'
+    #              end
+    #       [key,type]
+    #     }
+    #   ]
+    #   self.new(mapping, optional)
+    # end
+
+    def refer(data, strict=false)
+      field_names_key = FieldSet.field_names_key(data, self, strict, @waiting_fields)
       return @set_map[field_names_key] if @set_map.has_key?(field_names_key)
 
-      guessed = FieldSet.simple_guess(data)
+      guessed = self.simple_guess(data, false, strict)
       guessed_fields = guessed.fields
       @fields.each do |key,field|
         if guessed_fields.has_key?(key)
@@ -359,23 +240,25 @@ module Norikra
           guessed_fields[key] = field unless field.optional?
         end
       end
-      guessed.update_summary
+      guessed.update_summary #=> guessed
     end
 
-    def format(data)
-      # all keys of data should be already known at #format (before #format, do #refer)
-      ret = {}
-      data.each do |key, value|
-        ret[key] = @fields[key].format(value)
+    def dump_all
+      fields = {}
+      @fields.each do |key,field|
+        fields[field.name] = field.to_hash(true)
       end
-      ret
+      fields
     end
 
-    def dump
+    def dump # to cli display
       fields = {}
-      @fields.map{|key,field|
+      @fields.each do |key,field|
+        fields[key.to_sym] = field.to_hash(true) unless field.chained_access?
+      end
+      @container_fields.each do |key, field|
         fields[key.to_sym] = field.to_hash(true)
-      }
+      end
       fields
     end
   end