rtext 0.4.0 → 0.5.0

data/lib/rtext/frontend/context.rb

@@ -0,0 +1,56 @@
+module RText
+module Frontend
+
+module Context
+
+# lines; all lines from the beginning up to and including the current line
+def self.extract(lines)
+  non_ignored_lines = 0
+  array_nesting = 0
+  block_nesting = 0
+  last_element_line = 0
+  result = []
+  lines.reverse.each_with_index do |l, i|
+    if i == 0
+      result.unshift(l)
+    else
+      l = l.strip
+      if l.size == 0 || l[0..0] == "#"
+        # ignore empty lines and comments
+      else
+        non_ignored_lines += 1
+        case l[-1..-1]
+        when "{"
+          if block_nesting > 0
+            block_nesting -= 1 
+          elsif block_nesting == 0
+            result.unshift(l)
+            last_element_line = non_ignored_lines
+          end
+        when "}"
+          block_nesting += 1
+        when "["
+          if array_nesting > 0
+            array_nesting -= 1
+          elsif array_nesting == 0
+            result.unshift(l)
+          end
+        when "]"
+          array_nesting += 1
+        when ":"
+          # lable directly above element
+          if non_ignored_lines == last_element_line + 1
+            result.unshift(l)
+          end
+        end
+      end
+    end
+  end
+  result
+end
+
+end
+
+end
+end
+

data/lib/rtext/generic.rb

@@ -0,0 +1,13 @@
+require 'rgen/metamodel_builder'
+
+module RText
+
+class Generic < RGen::MetamodelBuilder::MMGeneric
+  attr_reader :string
+  def initialize(string)
+    @string = string
+  end
+end
+
+end
+

data/lib/rtext/instantiator.rb

@@ -36,6 +36,10 @@ class Instantiator
   #  :fragment_ref
   #    object which references the fragment being instantiated, will be set on model elements 
   #
+  #  :on_progress
+  #    a proc which is called twice for each model element: 
+  #    when the command token is recognized and when the element is instantiated 
+  #
   def instantiate(str, options={})
     @line_numbers = {}
     @env = options[:env]
@@ -44,6 +48,7 @@ class Instantiator
     @root_elements = options[:root_elements] || []
     @file_name = options[:file_name]
     @fragment_ref = options[:fragment_ref]
+    @on_progress_proc = options[:on_progress]
     @context_class_stack = []
     parser = Parser.new(@lang.reference_regexp)
     @root_elements.clear
@@ -64,15 +69,13 @@ class Instantiator
           unassociated_comments(args[3])
         end
       end,
+      :on_command_token => @on_progress_proc && lambda do
+        @on_progress_proc.call
+      end,
       :problems => parser_problems)
     parser_problems.each do |p|
       problem(p.message, p.line)
     end
-    if @unresolved_refs
-      @unresolved_refs.each do |ur|
-        ur.proxy.targetIdentifier = @lang.qualify_reference(ur.proxy.targetIdentifier, ur.element)
-      end
-    end
   end
 
   private
@@ -83,8 +86,9 @@ class Instantiator
     end
   end
 
-  def create_element(command, arg_list, element_list, comments, is_root)
+  def create_element(command, arg_list, element_list, comments, annotation, is_root)
     clazz = @context_class_stack.last 
+    @on_progress_proc.call if @on_progress_proc
     if !@lang.has_command(command.value)
       problem("Unknown command '#{command.value}'", command.line)
       return
@@ -132,6 +136,7 @@ class Instantiator
     comments.each do |c|
       handle_comment(c, element)
     end
+    handle_annotation(annotation, element) if annotation
     element
   end
 
@@ -209,7 +214,13 @@ class Instantiator
     end
     expected_kind = expected_token_kind(feature)
     value.each do |v|
-      if !expected_kind.include?(v.kind)
+      if v.kind == :generic
+        if @lang.generics_enabled
+          element.setOrAddGeneric(feature.name, v.value)
+        else
+          problem("Generic value not allowed", line)
+        end
+      elsif !expected_kind.include?(v.kind)
         problem("Argument '#{name}' can not take a #{v.kind}, expected #{expected_kind.join(", ")}", line)
       elsif feature.eType.is_a?(RGen::ECore::EEnum) 
         if !feature.eType.eLiterals.name.include?(v.value)
@@ -250,6 +261,18 @@ class Instantiator
     end
   end
 
+  def handle_annotation(annotation_desc, element)
+    if @lang.annotation_handler
+      annotation = annotation_desc.collect{|c| c.value}.join("\n")
+      success = @lang.annotation_handler.call(annotation, element, @env)
+      if !success 
+        problem("Annotation not allowed", line_number(element))
+      end
+    else
+      problem("Annotation not allowed", line_number(element))
+    end
+  end
+
   def is_labeled(a)
     a.is_a?(Array) && a[0].respond_to?(:kind) && a[0].kind == :label
   end

data/lib/rtext/language.rb

@@ -44,12 +44,14 @@ class Language
   #     default: word characters separated by at least one slash (/) 
   #
   #  :identifier_provider
-  #     a Proc which receives an element and its containing element or nil and should return 
-  #     the element's identifier as a string
+  #     a Proc which receives an element, its containing element, the feature through which the
+  #     element is referenced and the index position of the reference within the feature's values.
+  #     the latter 3 argumnets may be nil. it should return the element's identifier as a string.
   #     the identifier must be unique for the element unless "per_type_identifier" is set to true,
-  #     in which case they must be unique for each element of the same type
-  #     identifiers may be relative to the given containing element. in this case a globally unique 
-  #     identifer must be resonstructed by the proc specified using the :reference_qualifier option.
+  #     in which case they must be unique for each element of the same type.
+  #     identifiers may be relative to the given containing element, depending on the given
+  #     feature and index position. in this case a globally unique 
+  #     identifier must be resonstructed by the proc specified using the :reference_qualifier option.
   #     if the containing element is nil, the identifier returned must be globally unique.
   #     default: identifiers calculated by QualifiedNameProvider
   #              in this case options to QualifiedNameProvider may be provided and will be passed through
@@ -59,16 +61,11 @@ class Language
   #     default: false
   #
   #  :reference_qualifier
-  #     a Proc which receives an element identifier as returned by the identifier provider and 
-  #     another element which uses this identifier to reference the element.
-  #     it must return the globally unique version of the identifier.
-  #     in case the received identifier is already globally unique, it must be returned as is.
-  #     the received element might only be similar to the original referencing element. the reason
-  #     is that this element may need to be constructed using only partially parsable data.
-  #     it is garantueed though that the element's chain of containing elements is complete and
-  #     that (non-containment) references are resolved as far as possible.
-  #     default: no reference qualifier, i.e. all identifiers returned by the identifier provider 
-  #              must be globally unique
+  #     a Proc which receives RGen unresolved references and either a FragmentedModel or a ModelFragment.
+  #     it should modify the unresolved references' targetIdentifiers to make them globally unique.
+  #     the Proc is called for each fragment after it as been loaded and for the overall model.
+  #     this can be used to qualify context specific references returned by the identifier provider.
+  #     default: no reference qualifier
   #
   #  :root_classes
   #     an Array of EClass objects representing classes which can be used on root level
@@ -98,6 +95,18 @@ class Language
   #     the Proc may also modify the element to remove information already part of the comment
   #     default: no comments
   #
+  #  :annotation_handler
+  #     a Proc which will be invoked when a new element has been instantiated. receives  
+  #     the annotation as a string, the element and the environment to which the element has been added to.
+  #     the environment may be nil. it may change the model or otherwise use the annotated information. 
+  #     if the element can take no annotation, it should return false, otherwise true.
+  #     default: no handling of annotations 
+  #
+  #  :annotation_provider
+  #     a Proc which receives an element and should return this element's annotation as a string or nil.
+  #     the Proc may also modify the element to remove information already part of the annotation.
+  #     default: no annotations
+  #
   #  :indent_string
   #     the string representing one indent, could be a tab or spaces
   #     default: 2 spaces
@@ -106,10 +115,22 @@ class Language
   #     a Proc which receives an EClass object and should return an RText command name
   #     default: class name
   #
+  #  :backward_ref_attribute
+  #     a Proc which receives an EClass object and should return the name of this class's
+  #     feature which is used to represent backward references (i.e. for following the backward
+  #     reference, the user must click on a value of this attribute)
+  #     a value of nil means that the command name is used to follow the backward reference
+  #     default: nil (command name)
+  #  
+  #  :enable_generics
+  #     if set to true, generics (<value>) are allowed, otherwise forbidden
+  #     default: false
+  #
   def initialize(root_epackage, options={})
     @root_epackage = root_epackage
     @feature_provider = options[:feature_provider] || 
-      proc { |c| RGen::Serializer::OppositeReferenceFilter.call(c.eAllStructuralFeatures) }
+      proc { |c| RGen::Serializer::OppositeReferenceFilter.call(c.eAllStructuralFeatures).
+        reject{|f| f.derived} }
     @unlabled_arguments = options[:unlabled_arguments]
     @unquoted_arguments = options[:unquoted_arguments]
     @argument_format_provider = options[:argument_format_provider]
@@ -118,31 +139,45 @@ class Language
     setup_commands(root_epackage, command_name_provider)
     @reference_regexp = options[:reference_regexp] || /\A\w*(\/\w*)+/
     @identifier_provider = options[:identifier_provider] || 
-      proc { |element, context|
+      proc { |element, context, feature, index|
         @qualified_name_provider ||= RGen::Serializer::QualifiedNameProvider.new(options)
-        @qualified_name_provider.identifier(element)
+        name_attribute = options[:attribute_name] || "name"
+        if element.is_a?(RGen::MetamodelBuilder::MMProxy) || element.respond_to?(name_attribute)
+          @qualified_name_provider.identifier(element)
+        else
+          nil
+        end
       }
-    @reference_qualifier = options[:reference_qualifier]
+    @reference_qualifier = options[:reference_qualifier] || proc{|urefs, model_or_fragment| }
     @line_number_attribute = options[:line_number_attribute]
     @file_name_attribute = options[:file_name_attribute]
     @fragment_ref_attribute = options[:fragment_ref_attribute]
     @comment_handler = options[:comment_handler]
     @comment_provider = options[:comment_provider]
+    @annotation_handler = options[:annotation_handler]
+    @annotation_provider = options[:annotation_provider]
     @indent_string = options[:indent_string] || "  "
     @per_type_identifier = options[:per_type_identifier]
+    @backward_ref_attribute = options[:backward_ref_attribute] || proc{|c| nil}
+    @generics_enabled = options[:enable_generics]
   end
 
   attr_reader :root_epackage
   attr_reader :root_classes
   attr_reader :reference_regexp
   attr_reader :identifier_provider
+  attr_reader :reference_qualifier
   attr_reader :line_number_attribute
   attr_reader :file_name_attribute
   attr_reader :fragment_ref_attribute
   attr_reader :comment_handler
   attr_reader :comment_provider
+  attr_reader :annotation_handler
+  attr_reader :annotation_provider
   attr_reader :indent_string
   attr_reader :per_type_identifier
+  attr_reader :backward_ref_attribute
+  attr_reader :generics_enabled
 
   def class_by_command(command, context_class)
     map = @class_by_command[context_class]
@@ -212,14 +247,6 @@ class Language
     @fragment_ref_attribute && element.respond_to?(@fragment_ref_attribute) && element.send(@fragment_ref_attribute)
   end
 
-  def qualify_reference(identifier, element)
-    if @reference_qualifier
-      @reference_qualifier.call(identifier, element)
-    else
-      identifier
-    end
-  end
-
   private
 
   def setup_commands(root_epackage, command_name_provider)

data/lib/rtext/link_detector.rb

@@ -0,0 +1,57 @@
+require 'rtext/tokenizer'
+require 'rtext/context_builder'
+
+module RText
+
+class LinkDetector
+include RText::Tokenizer
+
+def initialize(lang)
+  @lang = lang
+end
+
+LinkDesc = Struct.new(:element, :feature, :index, :backward, :value, :scol, :ecol)
+
+# column numbers start at 1
+def detect(lines, column)
+  # make sure there is a space at the end of the line
+  # otherwise an important attribute value (e.g. the local name) might be missing in the
+  # context model since the context builder removes it as "just being completed"
+  lines.last.concat(" ")
+  current_line = lines.last
+  context = ContextBuilder.build_context(@lang, lines, lines.last.size)
+  tokens = tokenize(lines.last, @lang.reference_regexp)
+  token = tokens.find{|t| t.scol && t.scol <= column && t.ecol && t.ecol >= column}
+  if context && context.element && token &&
+      [:reference, :integer, :string, :identifier].include?(token.kind)
+    if column > 1
+      line_prefix = lines.last[0..column-2]
+    else
+      line_prefix = ""
+    end
+    context2 = ContextBuilder.build_context(@lang, lines, line_prefix.size) 
+    # for command prefixes nested below a lable, the context element and feature are
+    # the parent element and feature, this is not what's expected here
+    if line_prefix =~ /^\s*\w*$/
+      context2.element = nil
+      context2.feature = nil
+    end
+    bwref_attr = @lang.backward_ref_attribute.call(context.element.class.ecore)
+    if bwref_attr
+      is_backward = (context2.feature && context2.feature.name == bwref_attr)
+    else
+      is_backward = (token.kind == :identifier && line_prefix =~ /^\s*\w*$/)
+    end
+    is_backward = is_backward ? true : false
+    if context2.element && context2.feature
+      index = context2.element.getGenericAsArray(context2.feature.name).size
+    end
+    LinkDesc.new(context.element, context2.feature, index, is_backward, token.value, token.scol, token.ecol)
+  else
+    nil
+  end
+end
+
+end
+
+end

data/lib/rtext/message_helper.rb

@@ -0,0 +1,77 @@
+require 'json'
+
+module RText
+
+module MessageHelper
+
+def serialize_message(obj)
+  each_json_object_string(obj) do |s|
+    s.force_encoding("binary")
+    bytes = s.bytes.to_a
+    s.clear
+    bytes.each do |b|
+      if b >= 128 || b == 0x25 # %
+        s << "%#{b.to_s(16)}".force_encoding("binary")
+      else
+        s << b.chr("binary")
+      end
+    end
+    # there are no non ascii-7-bit characters left
+    s.force_encoding("utf-8")
+  end
+  json = JSON(obj)
+  # the JSON method outputs data in UTF-8 encoding
+  # the RText protocol expects message lengths measured in bytes
+  # there shouldn't be any non-ascii-7-bit characters, though, so json.size would also be ok
+  "#{json.bytesize}#{json}"
+end
+
+def extract_message(data)
+  # interpret input data as binary
+  data.force_encoding("binary")
+  obj = nil
+  if data =~ /^(\d+)\{/
+    length_length = $1.size
+    length = $1.to_i
+    if data.size >= length_length + length
+      data.slice!(0..(length_length-1))
+      json = data.slice!(0..length-1)
+      # there shouldn't be any non ascii-7-bit characters, transcode just to be sure
+      # encode from binary to utf-8 with :undef => :replace turns all non-ascii-7-bit bytes
+      # into the replacement character (\uFFFD)
+      json.encode!("utf-8", :undef => :replace)
+      obj = JSON(json)
+    end
+  end
+  if obj
+    each_json_object_string(obj) do |s|
+      # change encoding back to binary 
+      # there could still be replacement characters (\uFFFD), turn them into "?"
+      s.encode!("binary", :undef => :replace)
+      s.gsub!(/%[0-9a-fA-F][0-9a-fA-F]/){|m| m[1..2].to_i(16).chr("binary")}
+      s.force_encoding("binary")
+    end
+  end
+  obj
+end
+
+def each_json_object_string(object, &block)
+  if object.is_a?(Hash)
+    object.each_pair do |k, v|
+      # don't call block for hash keys
+      each_json_object_string(v, &block)
+    end
+  elsif object.is_a?(Array)
+    object.each do |v|
+      each_json_object_string(v, &block)
+    end
+  elsif object.is_a?(String)
+    yield(object)
+  else
+    # ignore
+  end
+end
+
+end
+
+end

data/lib/rtext/parser.rb

@@ -1,6 +1,10 @@
+require 'rtext/generic'
+require 'rtext/tokenizer'
+
 module RText 
 
 class Parser
+  include RText::Tokenizer
 
   Problem = Struct.new(:message, :line)
 
@@ -14,7 +18,7 @@ class Parser
     @problems = options[:problems] || []
     @non_consume_count = 0
     @consume_problem_reported = false
-    @tokens = tokenize(str, @reference_regexp)
+    @tokens = tokenize(str, @reference_regexp, :on_command_token => options[:on_command_token])
     @last_line = @tokens.last && @tokens.last.line 
     #@debug = true
     begin
@@ -29,6 +33,7 @@ class Parser
   def parse_statement(is_root=false, allow_unassociated_comment=false)
     comments = [] 
     comment = parse_comment 
+    annotation = parse_annotation
     if (next_token && next_token == :identifier) || !allow_unassociated_comment
       comments << [ comment, :above] if comment
       command = consume(:identifier)
@@ -43,7 +48,7 @@ class Parser
         eol_comment = parse_eol_comment
         comments << [ eol_comment, :eol ] if eol_comment
         consume(:newline)
-        @asc_visitor.call(command, arg_list, element_list, comments, is_root)
+        @asc_visitor.call(command, arg_list, element_list, comments, annotation, is_root)
       else
         discard_until(:newline)
         nil
@@ -51,7 +56,7 @@ class Parser
     elsif comment
       # if there is no statement, the comment is non-optional
       comments << [ comment, :unassociated ]
-      @asc_visitor.call(nil, nil, nil, comments, nil)
+      @asc_visitor.call(nil, nil, nil, comments, nil, nil)
       nil
     else
       # die expecting an identifier (next token is not an identifier)
@@ -71,6 +76,16 @@ class Parser
     result
   end
 
+  def parse_annotation
+    result = nil
+    while next_token == :annotation
+      result ||= []
+      result << consume(:annotation)
+      consume(:newline)
+    end
+    result
+  end
+
   def parse_eol_comment
     if next_token == :comment
       consume(:comment)
@@ -169,7 +184,7 @@ class Parser
     result
   end
 
-  AnyValue = [:identifier, :integer, :float, :string, :boolean, :reference] 
+  AnyValue = [:identifier, :integer, :float, :string, :boolean, :reference, :generic] 
 
   def parse_value
     consume(*AnyValue)
@@ -238,70 +253,6 @@ class Parser
     end
   end
 
-  Token = Struct.new(:kind, :value, :line)
-   
-  def tokenize(str, reference_regexp)
-    result = []
-    str.split(/\r?\n/).each_with_index do |str, idx|
-      idx += 1
-      if str =~ /^\s*#(.*)/
-        result << Token.new(:comment, $1, idx) 
-      else
-        until str.empty?
-          case str
-          when reference_regexp
-            str = $'
-            result << Token.new(:reference, $&, idx)
-          when /\A[-+]?\d+\.\d+(?:e[+-]\d+)?\b/
-            str = $'
-            result << Token.new(:float, $&.to_f, idx)
-          when /\A0[xX][0-9a-fA-F]+\b/
-            str = $'
-            result << Token.new(:integer, $&.to_i(16), idx)
-          when /\A[-+]?\d+\b/
-            str = $'
-            result << Token.new(:integer, $&.to_i, idx)
-          when /\A"((?:[^"\\]|\\.)*)"/
-            str = $'
-            result << Token.new(:string, $1.
-              gsub('\\\\','\\').
-              gsub('\\"','"').
-              gsub('\\n',"\n").
-              gsub('\\r',"\r").
-              gsub('\\t',"\t").
-              gsub('\\f',"\f").
-              gsub('\\b',"\b"), idx)
-          when /\A(?:true|false)\b/
-            str = $'
-            result << Token.new(:boolean, $& == "true", idx)
-          when /\A([a-zA-Z_]\w*)\b(?:\s*:)?/
-            str = $'
-            if $&[-1] == ?: 
-              result << Token.new(:label, $1, idx)
-            else
-              result << Token.new(:identifier, $&, idx)
-            end
-          when /\A[\{\}\[\]:,]/
-            str = $'
-            result << Token.new($&, nil, idx)
-          when /\A#(.*)/
-            str = ""
-            result << Token.new(:comment, $1, idx)
-          when /\A\s+/
-            str = $'
-            # ignore
-          when /\A\S+/
-            str = $'
-            result << Token.new(:error, $&, idx)
-          end
-        end
-      end
-      result << Token.new(:newline, nil, idx) \
-        unless result.empty? || result.last.kind == :newline
-    end
-    result
-  end
-
 end
 
 end