rtext 0.3.0 → 0.4.0

data/CHANGELOG

@@ -14,3 +14,12 @@
 * Added result limit option to DefaultServiceProvider
 * Removed short_class_names option from Language
 
+=0.4.0
+
+* Made instantiator a lot more robust against parse errors
+* Added DefaultLoader option to not reload fragments with errors
+* Added service load progress indication and custom problem severity
+* Fixed serialization of enum literals starting with a digit
+* Fixed used port detection in service
+* Fixed completion option order to match order defined in lanugage
+

data/RText_Plugin_Implementation_Guide

@@ -188,13 +188,34 @@ The command implicitly reloads the model from the filesystem before checking for
     no parameters 
 
   Response Format:
-    result line n:       <file name>
-    result line n+1..n+m <line number>;<message>
+    result line 1:       <file name>
+    result line 2..n     <line number>;<message>
 
 Note that the file name is repeated only once for all problems within a file to reduce the amout of
 result data which needs to be transmitted.
 
 
+===Command: Show Problems 2
+
+Since protocol version 1
+
+Like the first version but provides progress information while the model is loaded and problem severity.
+
+  Request Format:
+    command id:          "show_problems2"
+    no parameters 
+
+  Response Format:
+    result line 1..n:    progress:<[0..100]>
+    result line n+1:     <file name>
+    result line n+2..m   <[diwef]>;<line number>;<message>
+
+Progress is expressed by a value between 0 and 100. The backend may choose to return any number
+of progress lines. It should start with 0 and end with 100 and progress should not go backwards.
+
+Problem severity is expressed with one of the following flags: d (debug), i (info), w (warn), e (error), f (fatal)
+
+
 ===Command: Reference Targets
 
 This command is used to retrieve the targets of a given reference. The reference location is expressed

data/Rakefile

@@ -8,7 +8,7 @@ DocFiles = [
 
 RTextGemSpec = Gem::Specification.new do |s|
   s.name = %q{rtext}
-  s.version = "0.3.0"
+  s.version = "0.4.0"
   s.date = Time.now.strftime("%Y-%m-%d")
   s.summary = %q{Ruby Textual Modelling}
   s.email = %q{martin dot thiede at gmx de}

data/lib/rtext/completer.rb

@@ -34,7 +34,7 @@ class Completer
         else
           # all target types which don't need a label
           # and all lables which are needed by a potential target type
-          clazz.eAllReferences.select{|r| r.containment}.each do |r|
+          @lang.containments(clazz).each do |r|
             ([r.eType] + r.eType.eAllSubTypes).select{|t| !t.abstract}.each do |t|
               if @lang.containments_by_target_type(clazz, t).size > 1
                 labled_refs << r
@@ -48,8 +48,7 @@ class Completer
           sort{|a,b| a.name <=> b.name}.collect do |c| 
             class_completion_option(c)
           end +
-        labled_refs.uniq.select{|r| r.name.index(context.prefix) == 0}.
-          sort!{|a,b| a.name <=> b.name}.collect do |r|
+        labled_refs.uniq.select{|r| r.name.index(context.prefix) == 0}.collect do |r|
             CompletionOption.new("#{r.name}:", "<#{r.eType.name}>")
           end
       else
@@ -86,16 +85,14 @@ class Completer
               context.element.getGenericAsArray(f.name).size > 0}
             result += @lang.unlabled_arguments(clazz).
               select{|f| f.name.index(context.prefix) == 0 && 
-                context.element.getGenericAsArray(f.name).empty?}[0..0].
-              sort{|a,b| a.name <=> b.name}.collect do |f| 
+                context.element.getGenericAsArray(f.name).empty?}[0..0].collect do |f| 
                 CompletionOption.new("<#{f.name}>", "<#{f.eType.name}>")
               end
           end
           # label completion
           result += @lang.labled_arguments(clazz).
             select{|f| f.name.index(context.prefix) == 0 && 
-              context.element.getGenericAsArray(f.name).empty?}.
-            sort{|a,b| a.name <=> b.name}.collect do |f| 
+              context.element.getGenericAsArray(f.name).empty?}.collect do |f| 
               CompletionOption.new("#{f.name}:", "<#{f.eType.name}>")
             end 
           result

data/lib/rtext/default_loader.rb

@@ -29,6 +29,10 @@ class DefaultLoader
   #  :cache
   #    a fragment cache to be used for loading
   #
+  #  :dont_reload_with_errors
+  #    if set to true, don't reload fragments which have parse errors 
+  #    instead keep the existing fragment but attach the new problem list
+  #
   def initialize(language, fragmented_model, options={})
     @lang = language
     @model = fragmented_model
@@ -40,6 +44,7 @@ class DefaultLoader
     @fragment_by_file = {}
     pattern = options[:pattern]
     @file_provider = options[:file_provider] || proc { Dir.glob(pattern) }
+    @dont_reload_with_errors = options[:dont_reload_with_errors]
   end
 
   # Loads or reloads model fragments from files using the file patterns or file provider 
@@ -48,16 +53,19 @@ class DefaultLoader
   #  :before_load
   #    a proc which is called before a file is actually loaded, receives the fragment to load
   #    into and a symbol indicating the kind of loading: :load, :load_cached, :load_update_cache
+  #    optionally, the proc may take third argument which is the overall number of files
   #    default: no before load proc
   # 
   #  :after_load
   #    a proc which is called after a file has been loaded, receives the fragment loaded
+  #    optionally, the proc may take second argument which is the overall number of files
   #    default: no after load proc
   #
   def load(options={})
     @before_load_proc = options[:before_load]
     @after_load_proc = options[:after_load]
     files = @file_provider.call 
+    @num_files = files.size
     @change_detector.check_files(files)
     @model.resolve(:fragment_provider => method(:fragment_provider),
       :use_target_type => @lang.per_type_identifier)
@@ -79,8 +87,18 @@ class DefaultLoader
   end
 
   def file_changed(file)
-    file_removed(file)
-    file_added(file)
+    fragment = RGen::Fragment::ModelFragment.new(file, 
+      :identifier_provider => @lang.identifier_provider)
+    load_fragment_cached(fragment)
+    if @dont_reload_with_errors && fragment.data[:problems].size > 0
+      # keep old fragment but attach new problems
+      old_fragment = @fragment_by_file[file]
+      old_fragment.data[:problems] = fragment.data[:problems] 
+    else
+      file_removed(file)
+      @model.add_fragment(fragment)
+      @fragment_by_file[file] = fragment
+    end
   end
 
   def fragment_provider(element)
@@ -101,18 +119,38 @@ class DefaultLoader
         end
       end
       if result == :invalid
-        @before_load_proc && @before_load_proc.call(fragment, :load_update_cache)
+        call_before_load_proc(fragment, :load_update_cache)
         load_fragment(fragment)
         @cache.store(fragment)
-      @after_load_proc && @after_load_proc.call(fragment)
+        call_after_load_proc(fragment)
       else
-        @before_load_proc && @before_load_proc.call(fragment, :load_cached)
-      @after_load_proc && @after_load_proc.call(fragment)
+        call_before_load_proc(fragment, :load_cached)
+        call_after_load_proc(fragment)
       end
     else
-      @before_load_proc && @before_load_proc.call(fragment, :load)
+      call_before_load_proc(fragment, :load)
       load_fragment(fragment)
-    @after_load_proc && @after_load_proc.call(fragment)
+      call_after_load_proc(fragment)
+    end
+  end
+
+  def call_before_load_proc(fragment, kind)
+    if @before_load_proc
+      if @before_load_proc.arity == 3
+        @before_load_proc.call(fragment, kind, @num_files) 
+      else
+        @before_load_proc.call(fragment, kind) 
+      end
+    end
+  end
+
+  def call_after_load_proc(fragment)
+    if @after_load_proc
+      if @after_load_proc.arity == 2
+        @after_load_proc.call(fragment, @num_files) 
+      else
+        @after_load_proc.call(fragment) 
+      end
     end
   end
 

data/lib/rtext/default_service_provider.rb

@@ -13,8 +13,12 @@ class DefaultServiceProvider
     })
   end
 
-  def load_model
-    @loader.load
+  def load_model(options={})
+    if options[:on_progress]
+      @loader.load(:after_load => options[:on_progress])
+    else
+      @loader.load
+    end
   end
 
   ReferenceCompletionOption = Struct.new(:identifier, :type)
@@ -81,8 +85,8 @@ class DefaultServiceProvider
 
   FileProblems = Struct.new(:file, :problems)
   Problem = Struct.new(:severity, :line, :message)
-  def get_problems
-    load_model
+  def get_problems(options={})
+    load_model(options)
     result = []
     @model.fragments.sort{|a,b| a.location <=> b.location}.each do |fragment|
       problems = []

data/lib/rtext/instantiator.rb

@@ -46,28 +46,27 @@ class Instantiator
     @fragment_ref = options[:fragment_ref]
     @context_class_stack = []
     parser = Parser.new(@lang.reference_regexp)
-    begin
-      @root_elements.clear
-      parser.parse(str, 
-        :descent_visitor => lambda do |command|
-          clazz = @lang.class_by_command(command.value, @context_class_stack.last)
-          # in case no class is found, nil will be pushed, this will case the next command
-          # lookup to act as if called from toplevel
-          @context_class_stack.push(clazz)
-        end,
-        :ascent_visitor => lambda do |*args|
-          if args[0]
-            element =create_element(*args)
-            @context_class_stack.pop
-            element
-          else
-            unassociated_comments(args[3])
-          end
-        end)
-    rescue Parser::Error => e
-      problem(e.message, e.line)
-      @unresolved_refs.clear if @unresolved_refs
-      @root_elements.clear
+    @root_elements.clear
+    parser_problems = []
+    parser.parse(str, 
+      :descent_visitor => lambda do |command|
+        clazz = @lang.class_by_command(command.value, @context_class_stack.last)
+        # in case no class is found, nil will be pushed, this will case the next command
+        # lookup to act as if called from toplevel
+        @context_class_stack.push(clazz)
+      end,
+      :ascent_visitor => lambda do |*args|
+        if args[0]
+          element =create_element(*args)
+          @context_class_stack.pop
+          element
+        else
+          unassociated_comments(args[3])
+        end
+      end,
+      :problems => parser_problems)
+    parser_problems.each do |p|
+      problem(p.message, p.line)
     end
     if @unresolved_refs
       @unresolved_refs.each do |ur|
@@ -115,7 +114,7 @@ class Instantiator
         if di_index < unlabled_args.size 
           set_argument(element, unlabled_args[di_index], a, defined_args, command.line)
           di_index += 1
-        else
+        elsif a != nil
           problem("Unexpected unlabled argument, #{unlabled_args.size} unlabled arguments expected", command.line)
         end
       end
@@ -203,6 +202,7 @@ class Instantiator
       return
     end
     value = [value] unless value.is_a?(Array)
+    value.compact!
     if value.size > 1 && !feature.many
       problem("Argument '#{name}' can take only one value", line)
       return

data/lib/rtext/parser.rb

@@ -2,6 +2,8 @@ module RText
 
 class Parser
 
+  Problem = Struct.new(:message, :line)
+
   def initialize(reference_regexp)
     @reference_regexp = reference_regexp
   end
@@ -9,10 +11,17 @@ class Parser
   def parse(str, options)
     @dsc_visitor = options[:descent_visitor]
     @asc_visitor = options[:ascent_visitor]
+    @problems = options[:problems] || []
+    @non_consume_count = 0
+    @consume_problem_reported = false
     @tokens = tokenize(str, @reference_regexp)
     @last_line = @tokens.last && @tokens.last.line 
-    while next_token
-      parse_statement(true, true)
+    #@debug = true
+    begin
+      while next_token
+        statement = parse_statement(true, true)
+      end
+    rescue InternalError
     end
   end
 
@@ -23,17 +32,22 @@ class Parser
     if (next_token && next_token == :identifier) || !allow_unassociated_comment
       comments << [ comment, :above] if comment
       command = consume(:identifier)
-      @dsc_visitor.call(command)
-      arg_list = []
-      parse_argument_list(arg_list)
-      element_list = []
-      if next_token == "{"
-        parse_statement_block(element_list, comments)
+      if command
+        @dsc_visitor.call(command)
+        arg_list = []
+        parse_argument_list(arg_list)
+        element_list = []
+        if next_token == "{"
+          parse_statement_block(element_list, comments)
+        end
+        eol_comment = parse_eol_comment
+        comments << [ eol_comment, :eol ] if eol_comment
+        consume(:newline)
+        @asc_visitor.call(command, arg_list, element_list, comments, is_root)
+      else
+        discard_until(:newline)
+        nil
       end
-      eol_comment = parse_eol_comment
-      comments << [ eol_comment, :eol ] if eol_comment
-      consume(:newline)
-      @asc_visitor.call(command, arg_list, element_list, comments, is_root)
     elsif comment
       # if there is no statement, the comment is non-optional
       comments << [ comment, :unassociated ]
@@ -42,6 +56,8 @@ class Parser
     else
       # die expecting an identifier (next token is not an identifier)
       consume(:identifier)
+      discard_until(:newline)
+      nil
     end
   end
 
@@ -90,7 +106,8 @@ class Parser
     else
       eol_comment = parse_eol_comment
       comments << [ eol_comment, :eol ] if eol_comment
-      consume(:newline)
+      nl = consume(:newline)
+      discard_until(:newline) unless nl
       parse_statement
     end
   end
@@ -99,7 +116,8 @@ class Parser
     consume("[")
     eol_comment = parse_eol_comment
     comments << [ eol_comment, :eol ] if eol_comment
-    consume(:newline)
+    nl = consume(:newline)
+    discard_until(:newline) unless nl
     result = []
     while next_token && next_token != "]"
       statement = parse_statement(false, true)
@@ -114,7 +132,7 @@ class Parser
 
   def parse_argument_list(arg_list)
     first = true
-    while !["{", :comment, :newline].include?(next_token)
+    while (AnyValue + [",", "[", :label, :error]).include?(next_token)
       consume(",") unless first
       first = false
       parse_argument(arg_list)
@@ -142,7 +160,7 @@ class Parser
     consume("[")
     first = true
     result = []
-    while next_token != "]"
+    while (AnyValue + [",", :error]).include?(next_token)
       consume(",") unless first
       first = false
       result << parse_value
@@ -151,34 +169,71 @@ class Parser
     result
   end
 
+  AnyValue = [:identifier, :integer, :float, :string, :boolean, :reference] 
+
   def parse_value
-    consume(:identifier, :integer, :float, :string, :boolean, :reference)
+    consume(*AnyValue)
   end
 
   def next_token
     @tokens.first && @tokens.first.kind
   end
 
-  class Error < Exception
-    attr_reader :message, :line
-    def initialize(message, line)
-      @message, @line = message, line
+  def discard_until(kind)
+    t = @tokens.shift
+    if t
+      puts "discarding #{t.kind} #{t.value}" if @debug
+      while t.kind != kind
+        t = @tokens.shift
+        break unless t
+        puts "discarding #{t.kind} #{t.value}" if @debug
+      end
     end
   end
 
   def consume(*args)
-    t = @tokens.shift
+    t = @tokens.first
     if t.nil?
-      raise Error.new("Unexpected end of file, expected #{args.join(", ")}", @last_line)
+      @non_consume_count += 1
+      report_consume_problem("Unexpected end of file, expected #{args.join(", ")}", @last_line)
+      return nil
     end
     if args.include?(t.kind)
+      @tokens.shift
+      @consume_problem_reported = false
+      @non_consume_count = 0
+      puts "consuming #{t.kind} #{t.value}" if @debug
       t
     else
       if t.kind == :error
-        raise Error.new("Parse error on token '#{t.value}'", t.line)
+        @tokens.shift
+        @non_consume_count = 0
+        report_consume_problem("Parse error on token '#{t.value}'", t.line)
+        return nil
       else
         value = " '#{t.value}'" if t.value
-        raise Error.new("Unexpected #{t.kind}#{value}, expected #{args.join(", ")}", t.line)
+        @non_consume_count += 1
+        report_consume_problem("Unexpected #{t.kind}#{value}, expected #{args.join(", ")}", t.line)
+        return nil
+      end
+    end
+  end
+
+  class InternalError < Exception
+  end
+
+  def report_consume_problem(message, line)
+    problem = Problem.new(message, line)
+    if @non_consume_count > 100
+      # safety check, stop reoccuring problems to avoid endless loops
+      @problems << Problem.new("Internal error", line) 
+      puts [@problems.last.message, @problems.last.line].inspect if @debug
+      raise InternalError.new
+    else
+      if !@consume_problem_reported
+        @consume_problem_reported = true
+        @problems << problem 
+        puts [@problems.last.message, @problems.last.line].inspect if @debug
       end
     end
   end

data/lib/rtext/serializer.rb

@@ -133,7 +133,7 @@ class Serializer
           result << v.to_s
         end
       elsif feature.eType.is_a?(RGen::ECore::EEnum)
-        if v.to_s =~ /\W/
+        if v.to_s =~ /^\d|\W/
           result << "\"#{v.to_s.gsub("\\","\\\\\\\\").gsub("\"","\\\"").gsub("\n","\\n").
             gsub("\r","\\r").gsub("\t","\\t").gsub("\f","\\f").gsub("\b","\\b")}\""
         else

data/lib/rtext/service.rb

@@ -54,13 +54,23 @@ class Service
       cmd = lines.shift
       invocation_id = lines.shift
       response = nil
+      progress_index = 0
       case cmd
+      when "protocol_version"
+        response = ["1"]
       when "refresh"
         response = refresh(lines) 
       when "complete"
         response = complete(lines)
       when "show_problems"
         response = get_problems(lines)
+      when "show_problems2"
+        response = get_problems(lines, :with_severity => true, :on_progress => lambda do |frag, num_frags|
+          progress_index += 1
+          num_frags = 1 if num_frags < 1
+          progress = ["progress: #{progress_index*100/num_frags}"]
+          send_response(progress, invocation_id, socket, from, :incremental => true)
+        end)
       when "get_reference_targets"
         response = get_reference_targets(lines)
       when "get_elements"
@@ -79,7 +89,7 @@ class Service
 
   private
 
-  def send_response(response, invocation_id, socket, from)
+  def send_response(response, invocation_id, socket, from, options={})
     @logger.debug(response.inspect) if @logger
     loop do
       packet_lines = []
@@ -88,7 +98,7 @@ class Service
         size += response.first.size
         packet_lines << response.shift
       end
-      if response.size > 0
+      if options[:incremental] || response.size > 0
         packet_lines.unshift("more\n")
       else
         packet_lines.unshift("last\n")
@@ -104,7 +114,7 @@ class Service
     port = PortRangeStart
     begin
       socket.bind("localhost", port)
-    rescue Errno::EADDRINUSE
+    rescue Errno::EADDRINUSE, Errno::EAFNOSUPPORT
       port += 1
       retry if port <= PortRangeEnd
       raise
@@ -120,7 +130,7 @@ class Service
   def complete(lines)
     linepos = lines.shift.to_i
     context = ContextBuilder.build_context(@lang, lines, linepos)
-    @logger.debug("context element: #{@lang.identifier_provider.call(context.element, nil)}") if @logger
+    @logger.debug("context element: #{@lang.identifier_provider.call(context.element, nil)}") if context && @logger
     current_line = lines.pop
     current_line ||= ""
     options = @completer.complete(context, lambda {|ref| 
@@ -132,13 +142,13 @@ class Service
     }
   end
 
-  def get_problems(lines)
-    # TODO: severity
+  def get_problems(lines, options={})
     result = []
-    @service_provider.get_problems.each do |fp|
+    severity = options[:with_severity] ? "e;" : ""
+    @service_provider.get_problems(:on_progress => options[:on_progress]).each do |fp|
       result << fp.file+"\n"
       fp.problems.each do |p| 
-        result << "#{p.line};#{p.message}\n"
+        result << "#{severity}#{p.line};#{p.message}\n"
       end
     end
     result

data/test/completer_test.rb

@@ -41,10 +41,10 @@ def test_after_command
   END
   assert_options([
     ["<unlabled1>", "<EString>"],
+    ["text:", "<EString>"],
     ["nums:", "<EInt>"],
-    ["others:", "<TestNode>"],
     ["related:", "<TestNode>"],
-    ["text:", "<EString>"]
+    ["others:", "<TestNode>"]
   ], options)
 end
 
@@ -71,9 +71,9 @@ def test_after_labled_value
   TestNode nums: 1, |
   END
   assert_options([
-    ["others:", "<TestNode>"],
+    ["text:", "<EString>"],
     ["related:", "<TestNode>"],
-    ["text:", "<EString>"]
+    ["others:", "<TestNode>"]
   ], options)
 end
 
@@ -83,10 +83,10 @@ def test_after_unlabled_value
   END
   assert_options([
     ["<unlabled2>", "<EInt>"],
+    ["text:", "<EString>"],
     ["nums:", "<EInt>"],
-    ["others:", "<TestNode>"],
     ["related:", "<TestNode>"],
-    ["text:", "<EString>"]
+    ["others:", "<TestNode>"]
   ], options)
 end
 
@@ -95,10 +95,10 @@ def test_after_unlabled_value2
   TestNode "bla", 1, |
   END
   assert_options([
+    ["text:", "<EString>"],
     ["nums:", "<EInt>"],
-    ["others:", "<TestNode>"],
     ["related:", "<TestNode>"],
-    ["text:", "<EString>"]
+    ["others:", "<TestNode>"]
   ], options)
 end
 
@@ -107,9 +107,9 @@ def test_after_array
   TestNode nums: [1, 2], |
   END
   assert_options([
-    ["others:", "<TestNode>"],
+    ["text:", "<EString>"],
     ["related:", "<TestNode>"],
-    ["text:", "<EString>"]
+    ["others:", "<TestNode>"]
   ], options)
 end
 

data/test/instantiator_test.rb

@@ -12,7 +12,7 @@ class InstantiatorTest < Test::Unit::TestCase
   module TestMM
     extend RGen::MetamodelBuilder::ModuleExtension
     class TestNode < RGen::MetamodelBuilder::MMBase
-      SomeEnum = RGen::MetamodelBuilder::DataTypes::Enum.new([:A, :B, :'non-word*chars'])
+      SomeEnum = RGen::MetamodelBuilder::DataTypes::Enum.new([:A, :B, :'non-word*chars', :'2you'])
       has_attr 'text', String
       has_attr 'integer', Integer
       has_attr 'boolean', Boolean
@@ -199,7 +199,7 @@ class InstantiatorTest < Test::Unit::TestCase
       TestNode text: B
       TestNode a problem here 
     ), TestMM, :file_name => "some_file")
-    assert_equal ["some_file"], problems.collect{|p| p.file}
+    assert_equal "some_file", problems.first.file
   end
 
   def test_file_name_setter
@@ -323,6 +323,19 @@ class InstantiatorTest < Test::Unit::TestCase
     assert_model_simple(env)
   end
 
+  def test_no_newline_at_eof
+    env, problems = instantiate(%Q(
+      TestNode), TestMM)
+    assert_no_problems(problems)
+  end
+
+  def test_no_newline_at_eof2
+    env, problems = instantiate(%Q(
+      TestNode {
+      }), TestMM)
+    assert_no_problems(problems)
+  end
+
   #
   # references
   # 
@@ -585,7 +598,7 @@ class InstantiatorTest < Test::Unit::TestCase
       }
     ), TestMM2)
     assert_problems([
-      /unexpected }, expected identifier/i
+      /unexpected \}, expected identifier/i
     ], problems)
   end
 
@@ -707,6 +720,351 @@ class InstantiatorTest < Test::Unit::TestCase
     assert_problems([/command 'NonRootClass' can not be used on root level/i], problems)
   end
 
+  #
+  # problem recovery
+  #
+
+  def test_missing_value
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode nums: 1, text:
+      TestNode nums: 2, text: {
+        SubNode
+      }
+      TestNode text: ,nums: 3 {
+        SubNode
+      }
+      TestNode nums: , text: , bla: 
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 4, root_elements.size
+    assert_equal [1], root_elements[0].nums
+    assert_nil root_elements[0].text
+    assert_equal [2], root_elements[1].nums
+    assert_equal 1, root_elements[1].childs.size
+    assert_equal [3], root_elements[2].nums
+    assert_equal 1, root_elements[2].childs.size
+    assert_problems([
+      [/unexpected newline, expected.*integer/i, 2],
+      [/unexpected \{, expected.*integer/i, 3],
+      [/unexpected ,, expected.*integer/i, 6],
+      [/unexpected ,, expected.*integer/i, 9],
+      [/unexpected ,, expected.*integer/i, 9],
+      [/unexpected newline, expected.*integer/i, 9],
+      [/unknown argument 'bla'/i, 9],
+    ], problems)
+  end
+
+  def test_missing_comma
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode nums: 1 text: "bla"
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_equal [1], root_elements[0].nums
+    assert_equal "bla", root_elements[0].text
+    assert_problems([
+      /unexpected label .*, expected ,/i,
+    ], problems)
+  end
+
+  def test_missing_label
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode nums: 1 "bla"
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_equal [1], root_elements[0].nums
+    assert_problems([
+      /unexpected string 'bla', expected ,/i,
+      /unexpected unlabled argument/i
+    ], problems)
+  end
+
+  def test_unclosed_bracket
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode nums: [1, "bla"
+      TestNode nums: [1, text: "bla"
+      TestNode nums: [1 text: "bla"
+      TestNode nums: [1 "bla"
+      TestNode [1, "bla"
+      TestNode [1, "bla" [
+      TestNode [1, "bla", [
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 7, root_elements.size
+    assert_equal [1], root_elements[0].nums
+    assert_nil root_elements[0].text
+    assert_equal [1], root_elements[1].nums
+    assert_equal "bla", root_elements[1].text
+    assert_equal [1], root_elements[2].nums
+    assert_equal "bla", root_elements[2].text
+    assert_equal [1], root_elements[3].nums
+    assert_nil root_elements[3].text
+    assert_equal [], root_elements[4].nums
+    assert_nil root_elements[4].text
+    assert_problems([
+      [/unexpected newline, expected \]/i, 2],
+      [/argument 'nums' can not take a string, expected integer/i, 2],
+      [/unexpected label 'text', expected identifier/i, 3],
+      [/unexpected label 'text', expected \]/i, 4],
+      [/unexpected string 'bla', expected ,/i, 5],
+      [/argument 'nums' can not take a string, expected integer/i, 5],
+      [/unexpected newline, expected \]/i, 5],
+      [/unexpected newline, expected \]/i, 6],
+      [/unexpected unlabled argument/i, 6],
+      [/unexpected \[, expected \]/i, 7],
+      [/unexpected newline, expected \]/i, 7],
+      [/unexpected unlabled argument/i, 7],
+      [/unexpected unlabled argument/i, 7],
+      [/unexpected \[, expected identifier/i, 8],
+      [/unexpected unlabled argument/i, 8],
+      [/unexpected unlabled argument/i, 8],
+      [/unexpected newline, expected \]/i, 8],
+    ], problems)
+  end
+
+  def test_closing_bracket
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode ] 
+      TestNode 1 ] 
+      TestNode 1, ] 
+      TestNode nums: ]1, "bla"
+      TestNode text: "bla" ] 
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 5, root_elements.size
+    assert_equal [], root_elements[3].nums
+    assert_equal "bla", root_elements[4].text
+    assert_problems([
+      [/unexpected \], expected newline/i, 2],
+      [/unexpected \], expected newline/i, 3],
+      [/unexpected unlabled argument/i, 3],
+      [/unexpected \], expected identifier/i, 4],
+      [/unexpected unlabled argument/i, 4],
+      [/unexpected \], expected identifier/i, 5],
+      [/unexpected \], expected newline/i, 6],
+    ], problems)
+  end
+
+  def test_closing_brace
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode } 
+      TestNode 1 } 
+      TestNode 1, } 
+      TestNode nums: }1, "bla"
+      TestNode text: "bla" } 
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 5, root_elements.size
+    assert_equal [], root_elements[3].nums
+    assert_equal "bla", root_elements[4].text
+    assert_problems([
+      [/unexpected \}, expected newline/i, 2],
+      [/unexpected \}, expected newline/i, 3],
+      [/unexpected unlabled argument/i, 3],
+      [/unexpected \}, expected identifier/i, 4],
+      [/unexpected unlabled argument/i, 4],
+      [/unexpected \}, expected identifier/i, 5],
+      [/unexpected \}, expected newline/i, 6],
+    ], problems)
+  end
+
+  def test_starting_non_command
+    root_elements = []
+    env, problems = instantiate(%Q(
+      \)
+      TestNode
+      *
+      TestNode
+      $
+      TestNode
+      ,
+      TestNode
+      [
+      TestNode
+      {
+      TestNode
+      ]
+      TestNode
+      }
+      TestNode
+      }}
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 8, root_elements.size
+    assert_problems([
+      [/parse error on token '\)'/i, 2],
+      [/parse error on token '\*'/i, 4],
+      [/parse error on token '\$'/i, 6],
+      [/unexpected ,, expected identifier/i, 8],
+      [/unexpected \[, expected identifier/i, 10],
+      [/unexpected \{, expected identifier/i, 12],
+      [/unexpected \], expected identifier/i, 14],
+      [/unexpected \}, expected identifier/i, 16],
+      [/unexpected \}, expected identifier/i, 18],
+    ], problems)
+  end
+
+  def test_parse_error_in_argument_list
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode text: "bla", * nums: 1
+      TestNode text: "bla" * , nums: 1
+      TestNode ?text: "bla"
+      TestNode nums: [1, * 3]
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 4, root_elements.size
+    assert_equal "bla", root_elements[0].text
+    assert_equal [1], root_elements[0].nums
+    assert_equal "bla", root_elements[1].text
+    assert_equal [1], root_elements[1].nums
+    assert_equal [1, 3], root_elements[3].nums
+    assert_problems([
+      [/parse error on token '\*'/i, 2],
+      [/parse error on token '\*'/i, 3],
+      [/parse error on token '\?text:'/i, 4],
+      [/unexpected unlabled argument/i, 4],
+      [/parse error on token '\*'/i, 5],
+    ], problems)
+  end
+
+  def test_unclosed_brace
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode { 
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_problems([
+      [/unexpected end of file, expected \}/i, 2]
+    ], problems)
+  end
+
+  def test_unclosed_brace2
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode { 
+        *
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_problems([
+      [/parse error on token '\*'/i, 3]
+    ], problems)
+  end
+
+  def test_unclosed_brace3
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode { 
+        childs:
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_problems([
+      [/unexpected end of file, expected identifier/i, 3]
+    ], problems)
+  end
+
+  def test_label_without_child
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode { 
+        childs:
+      }
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_problems([
+      [/unexpected \}, expected identifier/i, 4]
+    ], problems)
+  end
+
+  def test_unclosed_bracket
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode { 
+        childs: [
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_problems([
+      [/unexpected end of file, expected \]/i, 3]
+    ], problems)
+  end
+
+  def test_child_label_problems
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode { 
+        childs: x 
+          SubNode
+        childs: * 
+          SubNode
+        childs: & 
+      }
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_equal 2, root_elements[0].childs.size
+    assert_problems([
+      [/unexpected identifier 'x', expected newline/i, 3],
+      [/parse error on token '\*'/i, 5],
+      [/parse error on token '&'/i, 7]
+    ], problems)
+  end
+
+  def test_child_label_problems_with_bracket
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode { 
+        childs: [ x 
+          SubNode
+        ]
+        childs: [ * 
+          SubNode
+        ]
+        childs: [& 
+        ]
+      }
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_equal 2, root_elements[0].childs.size
+    assert_problems([
+      [/unexpected identifier 'x', expected newline/i, 3],
+      [/parse error on token '\*'/i, 6],
+      [/parse error on token '&'/i, 9]
+    ], problems)
+  end
+
+  def test_missing_closing_bracket
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode { 
+        childs: [
+          SubNode
+        childs: [
+          SubNode
+        SubNode
+      }
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_equal 3, root_elements[0].childs.size
+    assert_problems([
+      [/unexpected label 'childs', expected identifier/i, 5],
+      [/unexpected \}, expected identifier/i, 8],
+    ], problems)
+  end
+
+  def test_missing_closing_brace
+    root_elements = []
+    env, problems = instantiate(%Q(
+      TestNode { 
+        TestNode {
+          TestNode
+      }
+    ), TestMM, :root_elements => root_elements)
+    assert_equal 1, root_elements.size
+    assert_equal 1, root_elements[0].childs.size
+    assert_equal 1, root_elements[0].childs[0].childs.size
+    assert_problems([
+      [/unexpected end of file, expected \}/i, 5],
+    ], problems)
+  end
+
   #
   # command name provider
   #
@@ -940,10 +1298,11 @@ class InstantiatorTest < Test::Unit::TestCase
         TestNode enum: A 
         TestNode enum: B 
         TestNode enum: "non-word*chars"
+        TestNode enum: "2you"
       }
     ), TestMM)
     assert_no_problems(problems)
-    assert_equal [:A, :B, :'non-word*chars'], env.find(:text => "root").first.childs.collect{|c| c.enum}
+    assert_equal [:A, :B, :'non-word*chars', :'2you'], env.find(:text => "root").first.childs.collect{|c| c.enum}
   end
 
   #
@@ -975,7 +1334,7 @@ class InstantiatorTest < Test::Unit::TestCase
         c.name == "TestNode" || c.name == "Data" || c.name == "TestNodeSub" || c.name == "SubNode"}))
     inst = RText::Instantiator.new(lang)
     problems = []
-    inst.instantiate(text, options.merge({:env => env, :problems => problems}))
+    inst.instantiate(text, options.merge({:env => env, :problems => problems, :root_elements => options[:root_elements]}))
     return env, problems
   end
   
@@ -987,9 +1346,15 @@ class InstantiatorTest < Test::Unit::TestCase
     remaining = problems.dup
     probs = []
     expected.each do |e|
-      p = problems.find{|p| p.message =~ e}
+      if e.is_a?(Array)
+        p = remaining.find{|p| p.message =~ e[0] && p.line == e[1]}
+      else
+        p = remaining.find{|p| p.message =~ e}
+      end
       probs << "expected problem not present: #{e}" if !p
-      remaining.delete(p)
+      # make sure to not delete duplicate problems at once
+      idx = remaining.index(p)
+      remaining.delete_at(idx) if idx
     end
     remaining.each do |p|
       probs << "unexpected problem: #{p.message}, line: #{p.line}"

data/test/serializer_test.rb

@@ -15,7 +15,7 @@ class SerializerTest < Test::Unit::TestCase
   module TestMM
     extend RGen::MetamodelBuilder::ModuleExtension
     SomeEnum = RGen::MetamodelBuilder::DataTypes::Enum.new(
-      :name => "SomeEnum", :literals => [:A, :B, :'non-word*chars'])
+      :name => "SomeEnum", :literals => [:A, :B, :'non-word*chars', :'2you'])
     class TestNode < RGen::MetamodelBuilder::MMBase
       has_attr 'text', String
       has_attr 'integer', Integer
@@ -355,7 +355,8 @@ TestNode {
     testModel = [
       TestMM::TestNode.new(:enum => :A),
       TestMM::TestNode.new(:enum => :B),
-      TestMM::TestNode.new(:enum => :'non-word*chars')
+      TestMM::TestNode.new(:enum => :'non-word*chars'),
+      TestMM::TestNode.new(:enum => :'2you')
     ]
     output = StringWriter.new
     serialize(testModel, TestMM, output) 
@@ -363,6 +364,7 @@ TestNode {
 TestNode enum: A
 TestNode enum: B
 TestNode enum: "non-word*chars"
+TestNode enum: "2you"
 ), output
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rtext
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-25 00:00:00.000000000Z
+date: 2012-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rgen
-  requirement: &22687308 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,12 @@ dependencies:
         version: 0.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *22687308
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.0
 description: RText can be used to derive textual languages from an RGen metamodel
   with very little effort.
 email: martin dot thiede at gmx de
@@ -79,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.7.2
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Ruby Textual Modelling