rtext 0.4.0 → 0.5.0

data/lib/rtext/serializer.rb

@@ -1,4 +1,5 @@
 require 'rtext/language'
+require 'rtext/generic'
 
 module RText
 
@@ -57,6 +58,13 @@ class Serializer
         write("##{l}")
       end
     end
+    # the annotation provider may modify the element
+    annotation = @lang.annotation_provider && @lang.annotation_provider.call(element)
+    if annotation
+      annotation.split(/\r?\n/).each do |l|
+        write("@#{l}")
+      end
+    end
     headline = @lang.command_by_class(clazz.instanceClass)
     raise "no command name for class #{clazz.instanceClass.to_s}" unless headline
     args = []
@@ -110,8 +118,15 @@ class Serializer
     values = element.getGenericAsArray(feature.name).compact
     result = []
     arg_format = @lang.argument_format(feature)
-    values.each do |v|
-      if feature.eType.instanceClass == Integer
+    values.each_with_index do |v, index|
+      if v.is_a?(RText::Generic)
+        str = v.string.split("%>").first
+        if str.index(">")
+          result << "<%#{str}%>"
+        else
+          result << "<#{str}>"
+        end
+      elsif feature.eType.instanceClass == Integer
         if arg_format 
           result << sprintf(arg_format, v)
         else
@@ -147,7 +162,7 @@ class Serializer
             gsub("\r","\\r").gsub("\t","\\t").gsub("\f","\\f").gsub("\b","\\b")}\""
         end
       elsif feature.is_a?(RGen::ECore::EReference)
-        result << @lang.identifier_provider.call(v, element) 
+        result << @lang.identifier_provider.call(v, element, feature, index) 
       end
     end
     if result.size > 1  

data/lib/rtext/service.rb

@@ -1,10 +1,14 @@
 require 'socket'
 require 'rtext/completer'
 require 'rtext/context_builder'
+require 'rtext/message_helper'
+require 'rtext/link_detector'
 
 module RText
 
 class Service
+  include RText::MessageHelper
+
   PortRangeStart = 9001
   PortRangeEnd   = 9100
 
@@ -18,170 +22,230 @@ class Service
   #  :logger
   #    a logger object on which the service will write its logging output
   #
-  def initialize(lang, service_provider, options={})
-    @lang = lang
+  #  :on_startup:
+  #    a Proc which is called right after the service has started up
+  #    can be used to output version information
+  #
+  def initialize(service_provider, options={})
     @service_provider = service_provider
-    @completer = RText::Completer.new(lang) 
     @timeout = options[:timeout] || 60
     @logger = options[:logger]
+    @on_startup = options[:on_startup]
   end
 
   def run
-    socket = create_socket 
-    puts "RText service, listening on port #{socket.addr[1]}"
+    server = create_server 
+    puts "RText service, listening on port #{server.addr[1]}"
+    @on_startup.call if @on_startup
     $stdout.flush
 
     last_access_time = Time.now
     last_flush_time = Time.now
-    stop_requested = false
-    while !stop_requested
+    @stop_requested = false
+    sockets = []
+    request_data = {}
+    while !@stop_requested
       begin
-        msg, from = socket.recvfrom_nonblock(65000)
-      rescue Errno::EWOULDBLOCK
-        sleep(0.01)
-        if (Time.now - last_access_time) > @timeout
-          @logger.info("RText service, stopping now (timeout)") if @logger
-          break 
+        sock = server.accept_nonblock
+        sock.sync = true
+        sockets << sock
+        @logger.info "accepted connection" if @logger
+      rescue Errno::EAGAIN, Errno::ECONNABORTED, Errno::EPROTO, Errno::EINTR, Errno::EWOULDBLOCK
+      rescue Exception => e
+        @logger.warn "unexpected exception during socket accept: #{e.class}"
+      end
+      sockets.dup.each do |sock|
+        data = nil
+        begin
+          data = sock.read_nonblock(100000)
+        rescue Errno::EWOULDBLOCK
+        rescue IOError, EOFError, Errno::ECONNRESET, Errno::ECONNABORTED
+          sock.close
+          request_data[sock] = nil
+          sockets.delete(sock)
+        rescue Exception => e
+          # catch Exception to make sure we don't crash due to unexpected exceptions
+          @logger.warn "unexpected exception during socket read: #{e.class}"
+          sock.close
+          request_data[sock] = nil
+          sockets.delete(sock)
         end
-        retry
+        if data
+          last_access_time = Time.now
+          request_data[sock] ||= ""
+          request_data[sock].concat(data)
+          while obj = extract_message(request_data[sock])
+            message_received(sock, obj)
+          end
+        end
+      end
+      IO.select([server] + sockets, [], [], 1)
+      if Time.now > last_access_time + @timeout
+        @logger.info("RText service, stopping now (timeout)") if @logger
+        break 
       end
       if Time.now > last_flush_time + FlushInterval
         $stdout.flush
         last_flush_time = Time.now
       end
-      last_access_time = Time.now
-      lines = msg.split(/\r?\n/)
-      cmd = lines.shift
-      invocation_id = lines.shift
-      response = nil
-      progress_index = 0
-      case cmd
-      when "protocol_version"
-        response = ["1"]
-      when "refresh"
-        response = refresh(lines) 
-      when "complete"
-        response = complete(lines)
-      when "show_problems"
-        response = get_problems(lines)
-      when "show_problems2"
-        response = get_problems(lines, :with_severity => true, :on_progress => lambda do |frag, num_frags|
-          progress_index += 1
-          num_frags = 1 if num_frags < 1
-          progress = ["progress: #{progress_index*100/num_frags}"]
-          send_response(progress, invocation_id, socket, from, :incremental => true)
-        end)
-      when "get_reference_targets"
-        response = get_reference_targets(lines)
-      when "get_elements"
-        response = get_open_element_choices(lines)
+    end
+  end
+
+  def message_received(sock, obj)
+    if check_request(obj) 
+      @logger.debug("request: "+obj.inspect) if @logger
+      response = { "type" => "response", "invocation_id" => obj["invocation_id"] }
+      case obj["command"]
+      when "load_model"
+        load_model(sock, obj, response)
+      when "content_complete"
+        content_complete(sock, obj, response)
+      when "link_targets" 
+        link_targets(sock, obj, response)
+      when "find_elements"
+        find_elements(sock, obj, response)
       when "stop"
-        response = [] 
         @logger.info("RText service, stopping now (stop requested)") if @logger
-        stop_requested = true
+        @stop_requested = true
       else
-        @logger.debug("unknown command #{cmd}") if @logger
-        response = []
+        @logger.warn("unknown command #{obj["command"]}") if @logger
+        response["type"] = "unknown_command_error"
+        response["command"] = obj["command"] 
       end
-      send_response(response, invocation_id, socket, from)
+      @logger.debug("response: "+response.inspect) if response && @logger
+      send_response(sock, response)
     end
   end
 
   private
 
-  def send_response(response, invocation_id, socket, from, options={})
-    @logger.debug(response.inspect) if @logger
-    loop do
-      packet_lines = []
-      size = 0
-      while response.size > 0 && size + response.first.size < 65000
-        size += response.first.size
-        packet_lines << response.shift
-      end
-      if options[:incremental] || response.size > 0
-        packet_lines.unshift("more\n")
-      else
-        packet_lines.unshift("last\n")
-      end
-      packet_lines.unshift("#{invocation_id}\n")
-      socket.send(packet_lines.join, 0, from[2], from[1])
-      break if response.size == 0 
+  def check_request(obj)
+    if obj["type"] != "request" 
+      @logger.warn("received message is not a request") if @logger
+      false
+    elsif !obj["invocation_id"].is_a?(Integer)
+      @logger.warn("invalid invocation id #{obj["invocation_id"]}") if @logger
+      false
+    else
+      true
     end
   end
 
-  def create_socket
-    socket = UDPSocket.new
-    port = PortRangeStart
-    begin
-      socket.bind("localhost", port)
-    rescue Errno::EADDRINUSE, Errno::EAFNOSUPPORT
-      port += 1
-      retry if port <= PortRangeEnd
-      raise
+  def load_model(sock, request, response)
+    problems = @service_provider.get_problems(
+      :on_progress => lambda do |frag, work_done, work_overall|
+        work_overall = 1 if work_overall < 1
+        work_done = work_overall if work_done > work_overall
+        work_done = 0 if work_done < 0
+        send_response(sock, {
+          "type" => "progress",
+          "invocation_id" => request["invocation_id"],
+          "percentage" => work_done*100/work_overall
+        })
+      end)
+    total = 0
+    response["problems"] = problems.collect do |fp|
+      { "file" => fp.file,
+        "problems" => fp.problems.collect do |p| 
+            total += 1
+            { "severity" => "error", "line" => p.line, "message" => p.message }
+          end }
     end
-    socket
-  end
-
-  def refresh(lines)
-    @service_provider.load_model
-    []
+    response["total_problems"] = total
   end
 
-  def complete(lines)
-    linepos = lines.shift.to_i
-    context = ContextBuilder.build_context(@lang, lines, linepos)
-    @logger.debug("context element: #{@lang.identifier_provider.call(context.element, nil)}") if context && @logger
-    current_line = lines.pop
-    current_line ||= ""
-    options = @completer.complete(context, lambda {|ref| 
+  def content_complete(sock, request, response)
+    # column numbers start at 1
+    linepos = request["column"]-1 
+    lines = request["context"]
+    lang = @service_provider.language
+    response["options"] = []
+    return unless lang
+    context = ContextBuilder.build_context(lang, lines, linepos)
+    @logger.debug("context element: #{lang.identifier_provider.call(context.element, nil, nil, nil)}") \
+      if context && context.element && @logger
+    completer = RText::Completer.new(lang) 
+    options = completer.complete(context, lambda {|ref| 
         @service_provider.get_reference_completion_options(ref, context).collect {|o|
           Completer::CompletionOption.new(o.identifier, "<#{o.type}>")}
       })
-    options.collect { |o|
-      "#{o.text};#{o.extra}\n"
-    }
-  end
-
-  def get_problems(lines, options={})
-    result = []
-    severity = options[:with_severity] ? "e;" : ""
-    @service_provider.get_problems(:on_progress => options[:on_progress]).each do |fp|
-      result << fp.file+"\n"
-      fp.problems.each do |p| 
-        result << "#{severity}#{p.line};#{p.message}\n"
-      end
+    response["options"] = options.collect do |o|
+      { "insert" => o.text, "display" => "#{o.text} #{o.extra}" }
     end
-    result
   end
 
-  def get_reference_targets(lines)
-    linepos = lines.shift.to_i
-    current_line = lines.last
-    context = ContextBuilder.build_context(@lang, lines, lines.last.size)
-    result = []
-    if context && current_line[linepos..linepos] =~ /[\w\/]/
-      ident_start = (current_line.rindex(/[^\w\/]/, linepos) || -1)+1
-      ident_end = (current_line.index(/[^\w\/]/, linepos) || current_line.size)-1
-      ident = current_line[ident_start..ident_end]
-      result << "#{ident_start};#{ident_end}\n"
-      if current_line[0..linepos+1] =~ /^\s*\w+$/
-        @service_provider.get_referencing_elements(ident, context).each do |t|
-          result << "#{t.file};#{t.line};#{t.display_name}\n"
+  def link_targets(sock, request, response)
+    # column numbers start at 1
+    linepos = request["column"]
+    lines = request["context"]
+    lang = @service_provider.language
+    response["targets"] = []
+    return unless lang
+    link_descriptor = RText::LinkDetector.new(lang).detect(lines, linepos)
+    if link_descriptor
+      response["begin_column"] = link_descriptor.scol
+      response["end_column"] = link_descriptor.ecol
+      targets = []
+      if link_descriptor.backward 
+        @service_provider.get_referencing_elements(
+            link_descriptor.value, link_descriptor.element, link_descriptor.feature, link_descriptor.index).each do |t|
+          targets << { "file" => t.file, "line" => t.line, "display" => t.display_name }
         end
       else
-        @service_provider.get_reference_targets(ident, context).each do |t|
-          result << "#{t.file};#{t.line};#{t.display_name}\n"
+        @service_provider.get_reference_targets(
+            link_descriptor.value, link_descriptor.element, link_descriptor.feature, link_descriptor.index).each do |t|
+          targets << { "file" => t.file, "line" => t.line, "display" => t.display_name }
         end
       end
+      response["targets"] = targets
+    end
+  end
+
+  def find_elements(sock, request, response)
+    pattern = request["search_pattern"] 
+    total = 0
+    response["elements"] = @service_provider.get_open_element_choices(pattern).collect do |c|
+      total += 1
+      { "display" => c.display_name, "file" => c.file, "line" => c.line }
+    end
+    response["total_elements"] = total
+  end
+
+  def send_response(sock, response)
+    if response
+      begin
+        sock.write(serialize_message(response))
+        sock.flush
+      # if there is an exception, the next read should shutdown the connection properly
+      rescue IOError, EOFError, Errno::ECONNRESET, Errno::ECONNABORTED
+      rescue Exception => e
+        # catch Exception to make sure we don't crash due to unexpected exceptions
+        @logger.warn "unexpected exception during socket write: #{e.class}"
+      end
     end
-    result
   end
 
-  def get_open_element_choices(lines)
-    pattern = lines.shift
-    @service_provider.get_open_element_choices(pattern).collect do |c|
-      "#{c.display_name};#{c.file};#{c.line}\n"
+  def create_server
+    port = PortRangeStart
+    serv = nil
+    begin
+      serv = TCPServer.new("localhost", port)
+      if serv.addr[0] == "AF_INET6"
+        # we need to make sure that two RText server won't connect on the same port but with IPv4 and IPv6
+        # therefore we always use the IPv4 interface, try again to get the IPv4 interface
+        # note that using "127.0.0.1" instead of localhost would always get us the IPv4 interface on the 
+        # first try, however, on some machines using the IP address instead of localhost doesn't work
+        # (EACCES error was seen, possibly due to some specific firewall setup)
+        serv2 = TCPServer.new("localhost", port)
+        serv.close
+        serv = serv2
+      end
+    rescue Errno::EADDRINUSE, Errno::EAFNOSUPPORT
+      port += 1
+      retry if port <= PortRangeEnd
+      raise
     end
+    serv
   end
 
 end

data/lib/rtext/tokenizer.rb

@@ -0,0 +1,102 @@
+require 'rtext/generic'
+
+module RText
+
+module Tokenizer
+
+  Token = Struct.new(:kind, :value, :line, :scol, :ecol)
+   
+  def tokenize(str, reference_regexp, options={})
+    result = []
+    on_command_token_proc = options[:on_command_token]
+    str.split(/\r?\n/).each_with_index do |str, idx|
+      idx += 1
+      if str =~ /^\s*([\#@])(.*)/
+        if $1 == "#"
+          result << Token.new(:comment, $2, idx, str.size-$2.size, str.size) 
+        else
+          result << Token.new(:annotation, $2, idx, str.size-$2.size, str.size) 
+        end
+      else
+        col = 1
+        first_token_in_line = true
+        until str.empty?
+          whitespace = false
+          case str
+          when reference_regexp
+            str = $'
+            result << Token.new(:reference, $&, idx, col, col+$&.size-1)
+            col += $&.size
+          when /\A[-+]?\d+\.\d+(?:e[+-]\d+)?\b/
+            str = $'
+            result << Token.new(:float, $&.to_f, idx, col, col+$&.size-1)
+            col += $&.size
+          when /\A0[xX][0-9a-fA-F]+\b/
+            str = $'
+            result << Token.new(:integer, $&.to_i(16), idx, col, col+$&.size-1)
+            col += $&.size
+          when /\A[-+]?\d+\b/
+            str = $'
+            result << Token.new(:integer, $&.to_i, idx, col, col+$&.size-1)
+            col += $&.size
+          when /\A"((?:[^"\\]|\\.)*)"/
+            str = $'
+            match = $&
+            result << Token.new(:string, $1.
+              gsub('\\\\','\\').
+              gsub('\\"','"').
+              gsub('\\n',"\n").
+              gsub('\\r',"\r").
+              gsub('\\t',"\t").
+              gsub('\\f',"\f").
+              gsub('\\b',"\b"), idx, col, col+match.size-1)
+            col += match.size
+          when /\A(?:true|false)\b/
+            str = $'
+            result << Token.new(:boolean, $& == "true", idx, col, col+$&.size-1)
+            col += $&.size
+          when /\A([a-zA-Z_]\w*)\b(?:\s*:)?/
+            str = $'
+            if $&[-1] == ?: 
+              result << Token.new(:label, $1, idx, col, col+$&.size-1)
+            else
+              result << Token.new(:identifier, $&, idx, col, col+$&.size-1)
+              if first_token_in_line && on_command_token_proc
+                on_command_token_proc.call
+              end
+            end
+            col += $&.size
+          when /\A[\{\}\[\]:,]/
+            str = $'
+            result << Token.new($&, nil, idx, col, col+$&.size-1)
+            col += $&.size
+          when /\A#(.*)/
+            str = ""
+            result << Token.new(:comment, $1, idx, col, col+$&.size-1)
+          when /\A\s+/
+            str = $'
+            col += $&.size
+            whitespace = true
+            # ignore
+          when /\A<%((?:(?!%>).)*)%>/, /\A<([^>]*)>/
+            str = $'
+            result << Token.new(:generic, RText::Generic.new($1), idx, col, col+$&.size-1)
+            col += $&.size
+          when /\A\S+/
+            str = $'
+            result << Token.new(:error, $&, idx, col, col+$&.size-1)
+            col += $&.size
+          end
+          first_token_in_line = false unless whitespace
+        end
+      end
+      result << Token.new(:newline, nil, idx) \
+        unless result.empty? || result.last.kind == :newline
+    end
+    result
+  end
+
+end
+
+end
+