rtext 0.7.0 → 0.8.0.pre1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dfdb25504469a30431bda0d53f6ed450809ca7ba
+  data.tar.gz: 93de80eedec085dfb25d6fb1d5f134236f692c57
+SHA512:
+  metadata.gz: dfe6ce9af736e941413be2ec99c4ad301ded2192affc6334dcf80eb795b2b9a98370b1174c385f852f0ba0788e9d2cf6db1130955eda6e845eca4758f5580e8d
+  data.tar.gz: 80f81b162444e33c346f5edacd961828c2311c9d99f1f404a94a62cac40a9f2dd6403608e7555da8f60210918f102cc92d308861dcf06ce871cdc2b0b00be79b

data/CHANGELOG

@@ -76,3 +76,7 @@
 * Minor performance improvements, mainly for instantiator and when passing large RText messages
 * Improved performance of frontend connector especially for large amounts of data
 
+=0.8.0
+
+* Added line breaks support
+

data/RText_Protocol

@@ -493,3 +493,11 @@ context in the frontend simple and the amount of data transmitted to the backend
 It's also a way to keep the parsing time of the context low in the backend and thus to minimize
 the user noticable delay.
 
+In case of line breaks, the frontend is responsible to join the lines before sending the 
+context information. For commands which use a column position, the position is the position
+within the joined line. This means that, when sending a command, the frontend must convert
+the column position in the broken line into the new position in the joined line.
+When reading back column information in a response (e.g. link command) the frontend must
+convert the column position in the joined line into a position in the respective broken
+fragment of a line.
+

data/Rakefile

@@ -8,7 +8,7 @@ DocFiles = [
 
 RTextGemSpec = Gem::Specification.new do |s|
   s.name = %q{rtext}
-  s.version = "0.7.0"
+  s.version = "0.8.0.pre1"
   s.date = Time.now.strftime("%Y-%m-%d")
   s.summary = %q{Ruby Textual Modelling}
   s.email = %q{martin dot thiede at gmx de}

data/lib/rtext/context_builder.rb

@@ -103,8 +103,10 @@ module ContextBuilder
     problem = nil
     line = context_lines.last
     if line =~ /\{\s*$/
-      # remove curly brace from last line, required for correct counting of num_elements
-      line.sub!(/\{\s*$/,"")
+      # remove curly brace from last line, required for correct counting of num_elements;
+      # also make sure that there is whitespace at the end of line, otherwise a word
+      # might get removed as "just being completed"
+      line.sub!(/\{\s*$/," ")
       problem = :after_curly
     end
     

data/lib/rtext/frontend/context.rb

@@ -1,10 +1,14 @@
 module RText
 module Frontend
 
-module Context
+class Context
 
-# lines; all lines from the beginning up to and including the current line
-def self.extract(lines)
+# lines: all lines from the beginning up to and including the current line
+# pos: position of the cursor in the last lines
+# returns the extracted lines and the new position in the last line
+def extract(lines, pos)
+  lines = filter_lines(lines)
+  lines, new_pos = join_lines(lines, pos)
   non_ignored_lines = 0
   array_nesting = 0
   block_nesting = 0
@@ -14,39 +18,67 @@ def self.extract(lines)
     if i == 0
       result.unshift(l)
     else
-      l = l.strip
-      if l.size == 0 || l[0..0] == "#"
-        # ignore empty lines and comments
-      else
-        non_ignored_lines += 1
-        case l[-1..-1]
-        when "{"
-          if block_nesting > 0
-            block_nesting -= 1 
-          elsif block_nesting == 0
-            result.unshift(l)
-            last_element_line = non_ignored_lines
-          end
-        when "}"
-          block_nesting += 1
-        when "["
-          if array_nesting > 0
-            array_nesting -= 1
-          elsif array_nesting == 0
-            result.unshift(l)
-          end
-        when "]"
-          array_nesting += 1
-        when ":"
-          # lable directly above element
-          if non_ignored_lines == last_element_line + 1
-            result.unshift(l)
-          end
+      non_ignored_lines += 1
+      case l.strip[-1..-1]
+      when "{"
+        if block_nesting > 0
+          block_nesting -= 1 
+        elsif block_nesting == 0
+          result.unshift(l)
+          last_element_line = non_ignored_lines
         end
+      when "}"
+        block_nesting += 1
+      when "["
+        if array_nesting > 0
+          array_nesting -= 1
+        elsif array_nesting == 0
+          result.unshift(l)
+        end
+      when "]"
+        array_nesting += 1
+      when ":"
+        # lable directly above element
+        if non_ignored_lines == last_element_line + 1
+          result.unshift(l)
+        end
+      end
+    end
+  end
+  [result, new_pos]
+end
+
+def filter_lines(lines)
+  lines.reject { |l| 
+    ls = l.strip
+    ls[0..0] == "@" || ls[0..0] == "#"
+  }
+end
+
+# when joining two lines, all whitespace after the last character of the first line is removed
+# (after , and [); however whitespace at the end of the last of several joined lines is preserved;
+# this way the context is correct even if the cursor is after the end of the last line 
+# (i.e. with whitespace after the last non-whitespace character)
+def join_lines(lines, pos)
+  outlines = []
+  while lines.size > 0
+    outlines << lines.shift
+    while lines.size > 0 && 
+        (outlines.last =~ /,\s*$/ || 
+        (outlines.last =~ /\[\s*$/ && outlines.last =~ /,/) ||
+        (lines.first =~ /^\s*\]/ && outlines.last =~ /,/))
+      outlines.last.rstrip!
+      l = lines.shift
+      if lines.size == 0
+        # strip only left part, the prefix might have whitespace on the
+        # right hand side which is relevant for the position
+        non_ws_prefix = l[0..pos-1].lstrip
+        pos = outlines.last.size + non_ws_prefix.size
       end
+      outlines.last.concat(l.lstrip)
     end
   end
-  result
+  [outlines, pos]
 end
 
 end

data/lib/rtext/language.rb

@@ -42,6 +42,16 @@ class Language
   #     (in sprintf syntax) which will be used by the serializer for integers and floats.
   #     default: if not present or the proc returns nil, then #to_s is used
   #
+  #  :newline_arguments
+  #     a Proc which receives an EClass and should return the names of this EClass's features
+  #     which are to be serialized after adding a line break.
+  #     default: no attributes on new lines
+  #
+  #  :newline_arrays
+  #     a Proc which receives an EClass and should return the names of this EClass's features
+  #     for which the array elements of their values are to be serialized after a line break.
+  #     default: no attributes on new lines
+  #
   #  :reference_regexp  
   #     a Regexp which is used by the tokenizer for identifying references 
   #     it must only match at the beginning of a string, i.e. it should start with \A
@@ -53,20 +63,25 @@ class Language
   #  :identifier_provider
   #     a Proc which receives an element, its containing element, the feature through which the
   #     element is referenced and the index position of the reference within the feature's values.
-  #     the latter 3 argumnets may be nil. it should return the element's identifier as a string.
-  #     the identifier must be unique for the element unless "per_type_identifier" is set to true,
+  #     the latter 3 arguments may be nil. it should return the element's identifier as a string.
+  #     the identifier must be unique for the element unless :per_type_identifier is set to true,
   #     in which case they must be unique for each element of the same type.
   #     identifiers may be relative to the given containing element, depending on the given
   #     feature and index position. in this case a globally unique 
-  #     identifier must be resonstructed by the proc specified using the :reference_qualifier option.
+  #     identifier must be reconstructed by the proc specified using the :reference_qualifier option.
   #     if the containing element is nil, the identifier returned must be globally unique.
-  #     default: identifiers calculated by QualifiedNameProvider
+  #     default: identifiers calculated by QualifiedNameProvider using the attribute provided using
+  #              the :attribute_name option
   #              in this case options to QualifiedNameProvider may be provided and will be passed through
   #
   #  :per_type_identifier
   #     if set to true, identifiers may be reused for elements of different type
   #     default: false
   #
+  #  :attribute_name
+  #     the name of the attribute used to calculate identifiers by the default identifier provider
+  #     default: "name"
+  #
   #  :reference_qualifier
   #     a Proc which receives RGen unresolved references and either a FragmentedModel or a ModelFragment.
   #     it should modify the unresolved references' targetIdentifiers to make them globally unique.
@@ -142,6 +157,8 @@ class Language
     @unquoted_arguments = options[:unquoted_arguments]
     @labeled_containments = options[:labeled_containments]
     @argument_format_provider = options[:argument_format_provider]
+    @newline_arguments = options[:newline_arguments]
+    @newline_arrays = options[:newline_arrays]
     @root_classes = options[:root_classes] || default_root_classes(root_epackage)
     command_name_provider = options[:command_name_provider] || proc{|c| c.name}
     setup_commands(root_epackage, command_name_provider)
@@ -232,6 +249,16 @@ class Language
     @argument_format_provider && @argument_format_provider.call(feature)
   end
 
+  def newline_argument?(clazz, feature)
+    return false unless @newline_arguments
+    @newline_arguments.call(clazz).include?(feature.name)
+  end
+
+  def newline_array?(clazz, feature)
+    return false unless @newline_arrays
+    @newline_arrays.call(clazz).include?(feature.name)
+  end
+
   def concrete_types(clazz)
     ([clazz] + clazz.eAllSubTypes).select{|c| !c.abstract}
   end
@@ -312,6 +339,8 @@ class Language
     :labled_arguments,
     :unquoted?,
     :labeled_containment?,
+    :newline_argument?,
+    :newline_array?,
     :argument_format,
     :concrete_types,
     :containments_by_target_type,

data/lib/rtext/parser.rb

@@ -130,7 +130,10 @@ class Parser
   def parse_argument_list(arg_list)
     first = true
     while (AnyValue + [",", "[", :label, :error]).include?(next_token_kind)
-      consume(",") unless first
+      unless first
+        success = consume(",")
+        consume(:newline) if success && next_token_kind == :newline
+      end
       first = false
       parse_argument(arg_list)
     end
@@ -155,13 +158,18 @@ class Parser
 
   def parse_argument_value_list
     consume("[")
+    consume(:newline) if next_token_kind == :newline
     first = true
     result = []
     while (AnyValue + [",", :error]).include?(next_token_kind)
-      consume(",") unless first
+      unless first
+        success = consume(",")
+        consume(:newline) if success && next_token_kind == :newline
+      end
       first = false
       result << parse_value
     end
+    consume(:newline) if next_token_kind == :newline && next_token_kind(1) == "]"
     consume("]")
     result
   end
@@ -172,8 +180,8 @@ class Parser
     consume(*AnyValue)
   end
 
-  def next_token_kind
-    @tokens.first && @tokens.first.kind
+  def next_token_kind(idx=0)
+    @tokens[idx] && @tokens[idx].kind
   end
 
   def discard_until(kind)

data/lib/rtext/serializer.rb

@@ -72,13 +72,24 @@ class Serializer
     args = []
     @lang.unlabled_arguments(clazz).each do |f|
       values = serialize_values(element, f)
-      args << values if values
+      args << [f, values] if values
     end
     @lang.labled_arguments(clazz).each do |f|
       values = serialize_values(element, f)
-      args << "#{f.name}: #{values}" if values
+      args << [f, "#{f.name}: #{values}"] if values
+    end
+    newline_arguments = false
+    args.each_with_index do |arg, index|
+      if @lang.newline_argument?(clazz, arg[0])
+        headline += " \\" if index == 0
+        headline += "\n" + @lang.indent_string * (@indent + 1)
+        newline_arguments = true
+      else
+        headline += " "
+      end
+      headline += arg[1]
+      headline += "," unless index == args.size-1
     end
-    headline += " "+args.join(", ") if args.size > 0
     contained_elements = {}
     @lang.containments(clazz).each do |f|
       contained_elements[f] = element.getGenericAsArray(f.name) 
@@ -87,6 +98,10 @@ class Serializer
       headline += " {"
       write(headline)
       iinc
+      # additional indentation needed if there are arguments on separate lines;
+      # note that this increment doesn't affect indentation of features of this element
+      # that have array values, because they have already been formatted in serialize_values
+      iinc if newline_arguments
       @lang.containments(clazz).each do |f|
         childs = contained_elements[f]
         if childs.size > 0
@@ -111,6 +126,7 @@ class Serializer
         end
       end
       idec
+      idec if newline_arguments
       write("}")
     else
       write(headline)
@@ -173,10 +189,17 @@ class Serializer
         result << @lang.identifier_provider.call(v, element, feature, index) 
       end
     end
-    if result.size > 1  
-      "[#{result.join(", ")}]"
+    if result.size > 1
+      if @lang.newline_array?(element.class.ecore, feature)
+        # inside an array, indent two steps further than the command
+        "[\n" + @lang.indent_string * (@indent + 2) +
+          result.join(",\n" + @lang.indent_string * (@indent + 2)) +
+          "\n" + @lang.indent_string * (@indent + 1) + "]"
+      else
+        "[#{result.join(", ")}]"
+      end
     elsif result.size == 1
-      result.first 
+      result.first
     else
       nil
     end

data/lib/rtext/tokenizer.rb

@@ -10,6 +10,7 @@ module Tokenizer
   def tokenize(str, reference_regexp, options={})
     result = []
     on_command_token_proc = options[:on_command_token]
+    linebreak = false
     str.split(/\r?\n/).each_with_index do |str, idx|
       idx += 1
       if idx == 1
@@ -27,7 +28,12 @@ module Tokenizer
         end
       else
         col = 1
-        first_token_in_line = true
+        if linebreak
+          # do not regard as the first token, if previous line ended in a linebreak
+          linebreak = false
+        else
+          first_token_in_line = true
+        end
         until str.empty?
           whitespace = false
           case str
@@ -94,6 +100,10 @@ module Tokenizer
             col += $&.size
             whitespace = true
             # ignore
+          when /\A\\\s*\Z/
+            str = $'
+            linebreak = true
+            # ignore
           when /\A<%((?:(?!%>).)*)%>/, /\A<([^>]*)>/
             str = $'
             result << Token.new(:generic, RText::Generic.new($1), idx, col, col+$&.size-1)
@@ -107,7 +117,7 @@ module Tokenizer
         end
       end
       result << Token.new(:newline, nil, idx) \
-        unless result.empty? || result.last.kind == :newline
+        unless linebreak || result.empty? || result.last.kind == :newline
     end
     result
   end

data/test/context_builder_test.rb

@@ -397,6 +397,14 @@ TestNode {|
   assert(c.element.is_a?(TestMM::TestNode))
 end
 
+def test_root_after_curly_no_ws
+  c = build_context TestMM, <<-END
+TestNode{|
+  END
+  assert_context c, :prefix => "", :feature => nil, :in_array => false, :in_block => false, :problem => :after_curly
+  assert(c.element.is_a?(TestMM::TestNode))
+end
+
 def test_in_cmd_after_cmd
   c = build_context TestMM, <<-END
   TestNode text: a {

data/test/frontend/context_test.rb

@@ -0,0 +1,205 @@
+$:.unshift File.join(File.dirname(__FILE__),"..","..","lib")
+
+require 'test/unit'
+require 'rtext/frontend/context'
+
+class ContextTest < Test::Unit::TestCase
+
+def test_simple
+  assert_context(
+    %Q(
+      A {
+        B {
+          |F bla
+    ),
+    %Q(
+      A {
+        B {
+          C a1: v1, a2: "v2"
+          D {
+            E a1: 5
+          }
+          |F bla
+    ))
+end
+
+def test_child_label
+  assert_context(
+    %Q(
+      A {
+        sub:
+          B {
+            F bla|
+    ),
+    %Q(
+      A {
+        sub:
+          B {
+            C a1: v1, a2: "v2"
+            D {
+              E a1: 5
+            }
+            F bla|
+    ))
+end
+
+def test_child_label_array
+  assert_context(
+    %Q(
+      A {
+        sub: [
+          B {
+            F| bla
+    ),
+    %Q(
+      A {
+        sub: [
+          B {
+            C
+          }
+          B {
+            C a1: v1, a2: "v2"
+            D {
+              E a1: 5
+            }
+            F| bla
+    ))
+end
+
+def test_ignore_child_lables
+  assert_context(
+    %Q(
+      A {
+        B {
+          F bl|a
+    ),
+    %Q(
+      A {
+        B {
+          sub:
+            C a1: v1, a2: "v2"
+          sub2: [
+            D {
+              E a1: 5
+            }
+          ]
+          F bl|a
+    ))
+end
+
+def test_linebreak
+  assert_context(
+    %Q(
+      A {
+        B {
+          C name,a1: v1,a2: "v2"|
+    ),
+    %Q(
+      A {
+        B {
+          C name,
+            a1: v1,
+            a2: "v2"|
+    ))
+end
+
+def test_linebreak_arg_array
+  assert_context(
+    %Q(
+      A {
+        B {
+          C name,a1: [v1,v2],a2: |5
+    ),
+    %Q(
+      A {
+        B {
+          C name,
+            a1: [   
+              v1,
+              v2
+            ], 
+            a2: |5
+    ))
+end
+
+def test_linebreak_empty_last_line
+  assert_context(
+    %Q(
+      A {
+        B name,|
+    ),
+    %Q(
+      A {
+        B name,
+          |
+    ))
+end
+
+def test_linebreak_empty_last_line2
+  assert_context(
+    %Q(
+      A {
+        B name,|
+    ),
+    %Q(
+      A {
+        B name,
+ |
+    ))
+end
+
+def test_linebreak_empty_lines
+  assert_context(
+    %Q(
+      A {
+        B name,a1: |
+    ),
+    %Q(
+      A {
+        B name, 
+
+        a1: |
+    ))
+end
+
+def test_comment_annotation
+  assert_context(
+    %Q(
+      A {
+        B {
+          |F bla
+    ),
+    %Q(
+      A {
+        # bla
+        B {
+          C a1: v1, a2: "v2"
+          # bla
+          D {
+            E a1: 5
+          }
+          @ anno
+          |F bla
+    ))
+end
+
+def assert_context(expected, text)
+  # remove first and last lines
+  # these are empty because of the use of %Q
+  exp_lines = expected.split("\n")[1..-2]
+  exp_col = exp_lines.last.index("|")
+  exp_lines.last.sub!("|","")
+  in_lines = text.split("\n")[1..-2]
+  in_col = in_lines.last.index("|")
+  in_lines.last.sub!("|","")
+  ctx = RText::Frontend::Context.new
+  lines, out_col = ctx.extract(in_lines, in_col)
+  assert_equal exp_lines, lines
+  if exp_col && in_col
+    assert_equal exp_col, out_col
+  end
+end
+
+end
+
+