rtext 0.5.1 → 0.7.0

data/lib/rtext/parser.rb

@@ -1,28 +1,21 @@
 require 'rtext/generic'
-require 'rtext/tokenizer'
 
 module RText 
 
 class Parser
-  include RText::Tokenizer
-
   Problem = Struct.new(:message, :line)
 
-  def initialize(reference_regexp)
-    @reference_regexp = reference_regexp
-  end
-
-  def parse(str, options)
+  def parse(tokens, options)
     @dsc_visitor = options[:descent_visitor]
     @asc_visitor = options[:ascent_visitor]
     @problems = options[:problems] || []
     @non_consume_count = 0
     @consume_problem_reported = false
-    @tokens = tokenize(str, @reference_regexp, :on_command_token => options[:on_command_token])
+    @tokens = tokens
     @last_line = @tokens.last && @tokens.last.line 
     #@debug = true
     begin
-      while next_token
+      while next_token_kind
         statement = parse_statement(true, true)
       end
     rescue InternalError
@@ -34,7 +27,7 @@ class Parser
     comments = [] 
     comment = parse_comment 
     annotation = parse_annotation
-    if (next_token && next_token == :identifier) || !allow_unassociated_comment
+    if (next_token_kind && next_token_kind == :identifier) || !allow_unassociated_comment
       comments << [ comment, :above] if comment
       command = consume(:identifier)
       if command
@@ -42,12 +35,10 @@ class Parser
         arg_list = []
         parse_argument_list(arg_list)
         element_list = []
-        if next_token == "{"
+        if next_token_kind == "{"
           parse_statement_block(element_list, comments)
         end
-        eol_comment = parse_eol_comment
-        comments << [ eol_comment, :eol ] if eol_comment
-        consume(:newline)
+        parse_eol_comment(comments)
         @asc_visitor.call(command, arg_list, element_list, comments, annotation, is_root)
       else
         discard_until(:newline)
@@ -68,7 +59,7 @@ class Parser
 
   def parse_comment
     result = nil 
-    while next_token == :comment
+    while next_token_kind == :comment
       result ||= []
       result << consume(:comment)
       consume(:newline)
@@ -78,7 +69,7 @@ class Parser
 
   def parse_annotation
     result = nil
-    while next_token == :annotation
+    while next_token_kind == :annotation
       result ||= []
       result << consume(:annotation)
       consume(:newline)
@@ -86,27 +77,26 @@ class Parser
     result
   end
 
-  def parse_eol_comment
-    if next_token == :comment
-      consume(:comment)
-    else
-      nil
+  def parse_eol_comment(comments)
+    if next_token_kind == :comment
+      comment = consume(:comment)
+      comments << [comment, :eol]
     end
+    nl = consume(:newline)
+    discard_until(:newline) unless nl
   end
 
   def parse_statement_block(element_list, comments)
     consume("{")
-    eol_comment = parse_eol_comment
-    comments << [ eol_comment, :eol ] if eol_comment
-    consume(:newline)
-    while next_token && next_token != "}"
+    parse_eol_comment(comments)
+    while next_token_kind && next_token_kind != "}"
       parse_block_element(element_list, comments)
     end
     consume("}")
   end
 
   def parse_block_element(element_list, comments)
-    if next_token == :label
+    if next_token_kind == :label
       label = consume(:label)
       element_list << [label, parse_labeled_block_element(comments)]
     else
@@ -116,38 +106,30 @@ class Parser
   end
 
   def parse_labeled_block_element(comments)
-    if next_token == "["
+    if next_token_kind == "["
       parse_element_list(comments)
     else
-      eol_comment = parse_eol_comment
-      comments << [ eol_comment, :eol ] if eol_comment
-      nl = consume(:newline)
-      discard_until(:newline) unless nl
+      parse_eol_comment(comments)
       parse_statement
     end
   end
 
   def parse_element_list(comments)
     consume("[")
-    eol_comment = parse_eol_comment
-    comments << [ eol_comment, :eol ] if eol_comment
-    nl = consume(:newline)
-    discard_until(:newline) unless nl
+    parse_eol_comment(comments)
     result = []
-    while next_token && next_token != "]"
+    while next_token_kind && next_token_kind != "]"
       statement = parse_statement(false, true)
       result << statement if statement 
     end
     consume("]")
-    eol_comment = parse_eol_comment
-    comments << [ eol_comment, :eol ] if eol_comment
-    consume(:newline)
+    parse_eol_comment(comments)
     result
   end
 
   def parse_argument_list(arg_list)
     first = true
-    while (AnyValue + [",", "[", :label, :error]).include?(next_token)
+    while (AnyValue + [",", "[", :label, :error]).include?(next_token_kind)
       consume(",") unless first
       first = false
       parse_argument(arg_list)
@@ -155,7 +137,7 @@ class Parser
   end
 
   def parse_argument(arg_list)
-    if next_token == :label
+    if next_token_kind == :label
       label = consume(:label)
       arg_list << [label, parse_argument_value]
     else
@@ -164,7 +146,7 @@ class Parser
   end
 
   def parse_argument_value
-    if next_token == "["
+    if next_token_kind == "["
       parse_argument_value_list
     else
       parse_value
@@ -175,7 +157,7 @@ class Parser
     consume("[")
     first = true
     result = []
-    while (AnyValue + [",", :error]).include?(next_token)
+    while (AnyValue + [",", :error]).include?(next_token_kind)
       consume(",") unless first
       first = false
       result << parse_value
@@ -190,7 +172,7 @@ class Parser
     consume(*AnyValue)
   end
 
-  def next_token
+  def next_token_kind
     @tokens.first && @tokens.first.kind
   end
 

data/lib/rtext/serializer.rb

@@ -1,3 +1,4 @@
+require 'bigdecimal'
 require 'rtext/language'
 require 'rtext/generic'
 
@@ -47,6 +48,7 @@ class Serializer
   end
   
   def serialize_element(element)
+    # TODO: remove setting of fragment ref (doesn't belong into serializer)
     set_fragment_ref(element)
     set_line_number(element, @line_number) if @set_line_number
     clazz = element.class.ecore
@@ -89,7 +91,8 @@ class Serializer
         childs = contained_elements[f]
         if childs.size > 0
           child_classes = childs.collect{|c| c.class.ecore}.uniq
-          if child_classes.any?{|c| @lang.containments_by_target_type(element.class.ecore, c).size > 1}
+          if @lang.labeled_containment?(clazz, f) ||
+              child_classes.any?{|c| @lang.containments_by_target_type(element.class.ecore, c).size > 1}
             if childs.size > 1
               write("#{f.name}: [")
               iinc
@@ -142,13 +145,18 @@ class Serializer
       elsif feature.eType.instanceClass == RGen::MetamodelBuilder::DataTypes::Boolean
         result << v.to_s
       elsif feature.eType.instanceClass == Float
-        if arg_format 
-          result << sprintf(arg_format, v)
+        if v.is_a?(BigDecimal)
+          result << v.to_s("F")
+          # formatting not available for BigDecimals
         else
-          result << v.to_s
+          if arg_format 
+            result << sprintf(arg_format, v)
+          else
+            result << v.to_s
+          end
         end
       elsif feature.eType.is_a?(RGen::ECore::EEnum)
-        if v.to_s =~ /^\d|\W/
+        if v.to_s =~ /^\d|\W/ || v.to_s == "true" || v.to_s == "false"
           result << "\"#{v.to_s.gsub("\\","\\\\\\\\").gsub("\"","\\\"").gsub("\n","\\n").
             gsub("\r","\\r").gsub("\t","\\t").gsub("\f","\\f").gsub("\b","\\b")}\""
         else

data/lib/rtext/service.rb

@@ -1,9 +1,10 @@
 require 'socket'
-require 'rtext/completer'
 require 'rtext/context_builder'
 require 'rtext/message_helper'
 require 'rtext/link_detector'
 
+# optimization: garbage collect while service is idle
+
 module RText
 
 class Service
@@ -93,6 +94,7 @@ class Service
 
   def message_received(sock, obj)
     if check_request(obj) 
+      request_start = Time.now
       @logger.debug("request: "+obj.inspect) if @logger
       response = { "type" => "response", "invocation_id" => obj["invocation_id"] }
       case obj["command"]
@@ -112,8 +114,10 @@ class Service
         response["type"] = "unknown_command_error"
         response["command"] = obj["command"] 
       end
-      @logger.debug("response: "+response.inspect) if response && @logger
+      @logger.debug("response: "+truncate_response_for_debug_output(response).inspect) \
+        if response && @logger
       send_response(sock, response)
+      @logger.info("request complete (#{Time.now-request_start}s)")
     end
   end
 
@@ -154,6 +158,9 @@ class Service
     response["total_problems"] = total
   end
 
+  InsertString = "insert"
+  DisplayString = "display"
+
   def content_complete(sock, request, response)
     # column numbers start at 1
     linepos = request["column"]-1 
@@ -164,13 +171,11 @@ class Service
     context = ContextBuilder.build_context(lang, lines, linepos)
     @logger.debug("context element: #{lang.identifier_provider.call(context.element, nil, nil, nil)}") \
       if context && context.element && @logger
-    completer = RText::Completer.new(lang) 
-    options = completer.complete(context, lambda {|ref| 
-        @service_provider.get_reference_completion_options(ref, context).collect {|o|
-          Completer::CompletionOption.new(o.identifier, "<#{o.type}>")}
-      })
+    options = @service_provider.get_completion_options(context)
+    insert_str = "insert"
+    display_str = "display"
     response["options"] = options.collect do |o|
-      { "insert" => o.text, "display" => "#{o.text} #{o.extra}" }
+      { insert_str => o.text, display_str => "#{o.text} #{o.extra}" }
     end
   end
 
@@ -186,16 +191,8 @@ class Service
       response["begin_column"] = link_descriptor.scol
       response["end_column"] = link_descriptor.ecol
       targets = []
-      if link_descriptor.backward 
-        @service_provider.get_referencing_elements(
-            link_descriptor.value, link_descriptor.element, link_descriptor.feature, link_descriptor.index).each do |t|
-          targets << { "file" => t.file, "line" => t.line, "display" => t.display_name }
-        end
-      else
-        @service_provider.get_reference_targets(
-            link_descriptor.value, link_descriptor.element, link_descriptor.feature, link_descriptor.index).each do |t|
-          targets << { "file" => t.file, "line" => t.line, "display" => t.display_name }
-        end
+      @service_provider.get_link_targets(link_descriptor).each do |t|
+        targets << { "file" => t.file, "line" => t.line, "display" => t.display_name }
       end
       response["targets"] = targets
     end
@@ -225,6 +222,18 @@ class Service
     end
   end
 
+  def truncate_response_for_debug_output(response_hash)
+    result = {}
+    response_hash.each_pair do |k,v|
+      if v.is_a?(Array) && v.size > 100
+        result[k] = v[0..99] + ["<truncated>"]
+      else
+        result[k] = v
+      end
+    end
+    result
+  end
+
   def create_server
     port = PortRangeStart
     serv = nil

data/lib/rtext/tokenizer.rb

@@ -1,4 +1,5 @@
 require 'rtext/generic'
+require 'bigdecimal'
 
 module RText
 
@@ -11,6 +12,13 @@ module Tokenizer
     on_command_token_proc = options[:on_command_token]
     str.split(/\r?\n/).each_with_index do |str, idx|
       idx += 1
+      if idx == 1
+        # remove UTF-8 BOM if present
+        enc = str.encoding
+        str.force_encoding("binary")
+        str = str[3..-1] if str.index("\xEF\xBB\xBF".force_encoding("binary")) == 0
+        str.force_encoding(enc)
+      end
       if str =~ /^\s*([\#@])(.*)/
         if $1 == "#"
           result << Token.new(:comment, $2, idx, str.size-$2.size, str.size) 
@@ -29,7 +37,15 @@ module Tokenizer
             col += $&.size
           when /\A[-+]?\d+\.\d+(?:e[+-]\d+)?\b/
             str = $'
-            result << Token.new(:float, $&.to_f, idx, col, col+$&.size-1)
+            val = $&
+            # if string size isn't more than 16, Float precision will be sufficient
+            # otherwise use BigDecimal even though the value might still fit in a Float
+            if val.size <= 16
+              val = val.to_f
+            else
+              val = BigDecimal.new(val)
+            end
+            result << Token.new(:float, val, idx, col, col+$&.size-1)
             col += $&.size
           when /\A0[xX][0-9a-fA-F]+\b/
             str = $'
@@ -82,7 +98,7 @@ module Tokenizer
             str = $'
             result << Token.new(:generic, RText::Generic.new($1), idx, col, col+$&.size-1)
             col += $&.size
-          when /\A\S+/
+          when /\A\S/
             str = $'
             result << Token.new(:error, $&, idx, col, col+$&.size-1)
             col += $&.size

data/test/completer_test.rb

@@ -4,7 +4,7 @@ require 'test/unit'
 require 'rgen/metamodel_builder'
 require 'rtext/language'
 require 'rtext/context_builder'
-require 'rtext/completer'
+require 'rtext/default_completer'
 
 class CompleterTest < Test::Unit::TestCase
 
@@ -14,6 +14,8 @@ module TestMM
     has_attr 'text', String
     has_attr 'unlabled', String
   end
+  class TestNode3 < RGen::MetamodelBuilder::MMBase
+  end
   class TestNode < RGen::MetamodelBuilder::MMBase
     has_attr 'text', String
     has_attr 'unlabled1', String
@@ -24,6 +26,7 @@ module TestMM
     contains_many 'childs', TestNode, 'parent'
     contains_one 'child2RoleA', TestNode2, 'parentA'
     contains_many 'child2RoleB', TestNode2, 'parentB'
+    contains_one_uni 'testNode3', TestNode3
   end
   SomeEnum = RGen::MetamodelBuilder::DataTypes::Enum.new(
     :name => "SomeEnum", :literals => [:A, :B, :'non-word*chars'])
@@ -271,7 +274,7 @@ TestNode3 enum: |
   assert_options([
     ["A", "<SomeEnum>"],
     ["B", "<SomeEnum>"],
-    ["non-word*chars", "<SomeEnum>"]
+    ["\"non-word*chars\"", "<SomeEnum>"]
   ], options)
 end
 
@@ -282,7 +285,7 @@ TestNode3 enum: A|
   assert_options([
     ["A", "<SomeEnum>"],
     ["B", "<SomeEnum>"],
-    ["non-word*chars", "<SomeEnum>"]
+    ["\"non-word*chars\"", "<SomeEnum>"]
   ], options)
 end
 
@@ -308,8 +311,8 @@ def test_reference_value
   options = complete(TestMM, %Q(\
 TestNode related: |\
   ), lambda { |r| [
-    RText::Completer::CompletionOption.new("A", "a"),
-    RText::Completer::CompletionOption.new("B", "b") ] })
+    RText::DefaultCompleter::CompletionOption.new("A", "a"),
+    RText::DefaultCompleter::CompletionOption.new("B", "b") ] })
   assert_options([
     ["A", "a"],
     ["B", "b"],
@@ -320,8 +323,8 @@ def test_reference_value_part
   options = complete(TestMM, %Q(\
 TestNode related: /My/|\
   ), lambda { |r| [
-    RText::Completer::CompletionOption.new("/My/Target", "a"),
-    RText::Completer::CompletionOption.new("/MyOther/Target", "b") ] })
+    RText::DefaultCompleter::CompletionOption.new("/My/Target", "a"),
+    RText::DefaultCompleter::CompletionOption.new("/MyOther/Target", "b") ] })
   assert_options([
     ["/My/Target", "a"],
     ["/MyOther/Target", "b"]
@@ -340,8 +343,8 @@ def test_reference_value_in_array
   options = complete(TestMM, %Q(\
 TestNode others: |
 ), lambda { |r| [
-    RText::Completer::CompletionOption.new("A", "a"),
-    RText::Completer::CompletionOption.new("B", "b") ] })
+    RText::DefaultCompleter::CompletionOption.new("A", "a"),
+    RText::DefaultCompleter::CompletionOption.new("B", "b") ] })
   assert_options([
     ["A", "a"],
     ["B", "b"],
@@ -352,8 +355,8 @@ def test_reference_value_in_array_after_bracket
   options = complete(TestMM, %Q(\
 TestNode others: [|
 ), lambda { |r| [
-    RText::Completer::CompletionOption.new("A", "a"),
-    RText::Completer::CompletionOption.new("B", "b") ] })
+    RText::DefaultCompleter::CompletionOption.new("A", "a"),
+    RText::DefaultCompleter::CompletionOption.new("B", "b") ] })
   assert_options([
     ["A", "a"],
     ["B", "b"],
@@ -364,8 +367,8 @@ def test_reference_value_in_array_second_value
   options = complete(TestMM, %Q(\
 TestNode others: [xxx, |
 ), lambda { |r| [
-    RText::Completer::CompletionOption.new("A", "a"),
-    RText::Completer::CompletionOption.new("B", "b") ] })
+    RText::DefaultCompleter::CompletionOption.new("A", "a"),
+    RText::DefaultCompleter::CompletionOption.new("B", "b") ] })
   assert_options([
     ["A", "a"],
     ["B", "b"],
@@ -377,8 +380,8 @@ def test_reference_value_nested
 TestNode {
   TestNode SimpleState, others: [|/StatemachineMM/State]
 ), lambda { |r| [
-    RText::Completer::CompletionOption.new("A", "a"),
-    RText::Completer::CompletionOption.new("B", "b") ] })
+    RText::DefaultCompleter::CompletionOption.new("A", "a"),
+    RText::DefaultCompleter::CompletionOption.new("B", "b") ] })
   assert_options([
     ["A", "a"],
     ["B", "b"],
@@ -401,7 +404,8 @@ TestNode {
   assert_options([
     ["TestNode", "<unlabled1>, <unlabled2>"],
     ["child2RoleA:", "<TestNode2>"],
-    ["child2RoleB:", "<TestNode2>"]
+    ["child2RoleB:", "<TestNode2>"],
+    ["testNode3:", "<TestNode3>"]
   ], options)
 end
 
@@ -580,9 +584,22 @@ def complete(mm, text, ref_comp_option_provider=nil)
   lang = RText::Language.new(mm.ecore,
     :root_classes => mm.ecore.eAllClasses,
     :unlabled_arguments => lambda {|c| ["unlabled1", "unlabled2", "unlabled"]},
-    :unquoted_arguments => lambda {|c| c.name == "TestNode2" ? ["text", "unlabled"] : []})
+    :unquoted_arguments => lambda {|c| c.name == "TestNode2" ? ["text", "unlabled"] : []},
+    :labeled_containments => lambda {|c| ["testNode3"]})
   context = RText::ContextBuilder.build_context(lang, context_lines, pos_in_line)
-  RText::Completer.new(lang).complete(context, ref_comp_option_provider)
+  completer = RText::DefaultCompleter.new(lang)
+  class << completer
+    attr_accessor :ref_comp_option_provider
+    def reference_options(context)
+      if ref_comp_option_provider
+        ref_comp_option_provider.call(context.feature)
+      else
+        super
+      end
+    end
+  end
+  completer.ref_comp_option_provider = ref_comp_option_provider
+  completer.complete(context)
 end
 
 end