rtext 0.4.0 → 0.5.0

data/Rakefile

@@ -1,39 +1,39 @@
-require 'rake/gempackagetask'
-require 'rake/rdoctask'
+require 'rubygems/package_task'
+require 'rdoc/task'
 
 DocFiles = [
-  "README", "CHANGELOG", "MIT-LICENSE", 
+  "README.rdoc", "CHANGELOG", "MIT-LICENSE", 
   "RText_Users_Guide", 
-  "RText_Plugin_Implementation_Guide"]
+  "RText_Protocol"]
 
 RTextGemSpec = Gem::Specification.new do |s|
   s.name = %q{rtext}
-  s.version = "0.4.0"
+  s.version = "0.5.0"
   s.date = Time.now.strftime("%Y-%m-%d")
   s.summary = %q{Ruby Textual Modelling}
   s.email = %q{martin dot thiede at gmx de}
   s.homepage = %q{http://ruby-gen.org}
   s.description = %q{RText can be used to derive textual languages from an RGen metamodel with very little effort.}
   s.authors = ["Martin Thiede"]
-  s.add_dependency('rgen', '>= 0.6.0')
+  s.add_dependency('rgen', '>= 0.6.1')
   gemfiles = Rake::FileList.new
   gemfiles.include("{lib,test}/**/*")
   gemfiles.include(DocFiles)
   gemfiles.include("Rakefile") 
   gemfiles.exclude(/\b\.bak\b/)
   s.files = gemfiles
-  s.rdoc_options = ["--main", "README", "-x", "test"]
+  s.rdoc_options = ["--main", "README.rdoc", "-x", "test"]
   s.extra_rdoc_files = DocFiles
 end
 
-Rake::RDocTask.new do |rd|
-  rd.main = "README"
+RDoc::Task.new do |rd|
+  rd.main = "README.rdoc"
   rd.rdoc_files.include(DocFiles)
   rd.rdoc_files.include("lib/**/*.rb")
   rd.rdoc_dir = "doc"
 end
 
-RTextPackageTask = Rake::GemPackageTask.new(RTextGemSpec) do |p|
+RTextPackageTask = Gem::PackageTask.new(RTextGemSpec) do |p|
   p.need_zip = false
 end	
 

data/lib/rtext/completer.rb

@@ -23,6 +23,7 @@ class Completer
     clazz = context && context.element && context.element.class.ecore
     if clazz
       if context.in_block
+        # command and lable completion within block
         types = []
         labled_refs = []
         if context.feature
@@ -44,63 +45,68 @@ class Completer
             end
           end
         end
-        types.uniq.select{|c| c.name.index(context.prefix) == 0}.
+        types.uniq.
           sort{|a,b| a.name <=> b.name}.collect do |c| 
             class_completion_option(c)
           end +
-        labled_refs.uniq.select{|r| r.name.index(context.prefix) == 0}.collect do |r|
+        labled_refs.uniq.collect do |r|
             CompletionOption.new("#{r.name}:", "<#{r.eType.name}>")
           end
-      else
+      elsif !context.problem
+        result = []
         if context.feature
+          if context.after_label
+            description = "<#{context.feature.eType.name}>"
+          else
+            description = "[#{context.feature.name}] <#{context.feature.eType.name}>"
+          end
           # value completion
           if context.feature.is_a?(RGen::ECore::EAttribute) || !context.feature.containment
             if context.feature.is_a?(RGen::ECore::EReference)
               if ref_completion_option_provider
-                ref_completion_option_provider.call(context.feature)
+                result += ref_completion_option_provider.call(context.feature)
               else
-                []
+                # no options 
               end
             elsif context.feature.eType.is_a?(RGen::ECore::EEnum)
-              context.feature.eType.eLiterals.collect do |l|
-                CompletionOption.new("#{l.name}")
+              result += context.feature.eType.eLiterals.collect do |l|
+                CompletionOption.new("#{l.name}", "<#{context.feature.eType.name}>")
               end 
             elsif context.feature.eType.instanceClass == String
-              [ CompletionOption.new("\"\"") ]
+              if @lang.unquoted?(context.feature)
+                result += [ CompletionOption.new("#{context.feature.name.gsub(/\W/,"")}", description) ]
+              else
+                result += [ CompletionOption.new("\"\"", description) ]
+              end
             elsif context.feature.eType.instanceClass == Integer 
-              (0..4).collect{|i| CompletionOption.new("#{i}") }
+              result += (0..0).collect{|i| CompletionOption.new("#{i}", description) }
             elsif context.feature.eType.instanceClass == Float 
-              (0..4).collect{|i| CompletionOption.new("#{i}.0") }
+              result += (0..0).collect{|i| CompletionOption.new("#{i}.0", description) }
             elsif context.feature.eType.instanceClass == RGen::MetamodelBuilder::DataTypes::Boolean
-              [true, false].collect{|b| CompletionOption.new("#{b}") }
+              result += [true, false].collect{|b| CompletionOption.new("#{b}", description) }
             else
-              []
+              # no options 
             end
           else
             # containment reference, ignore
           end
-        else
-          result = []
-          if !@lang.labled_arguments(clazz).any?{|f| 
-              context.element.getGenericAsArray(f.name).size > 0}
-            result += @lang.unlabled_arguments(clazz).
-              select{|f| f.name.index(context.prefix) == 0 && 
-                context.element.getGenericAsArray(f.name).empty?}[0..0].collect do |f| 
-                CompletionOption.new("<#{f.name}>", "<#{f.eType.name}>")
-              end
-          end
+        end
+        if !context.after_label
           # label completion
           result += @lang.labled_arguments(clazz).
-            select{|f| f.name.index(context.prefix) == 0 && 
-              context.element.getGenericAsArray(f.name).empty?}.collect do |f| 
+            select{|f| 
+              !context.element.eIsSet(f.name)}.collect do |f| 
               CompletionOption.new("#{f.name}:", "<#{f.eType.name}>")
             end 
-          result
         end
+        result
+      else
+        # missing comma, after curly brace, etc.
+        []
       end
     elsif context
       # root classes
-      @lang.root_classes.select{|c| c.name.index(context.prefix) == 0}.
+      @lang.root_classes.
         sort{|a,b| a.name <=> b.name}.collect do |c| 
           class_completion_option(c)
         end 

data/lib/rtext/context_builder.rb

@@ -1,4 +1,5 @@
 require 'rtext/instantiator'
+require 'rtext/tokenizer'
 
 module RText
 
@@ -25,23 +26,33 @@ module RText
 #
 module ContextBuilder
 
-  Context = Struct.new(:element, :feature, :prefix, :in_array, :in_block)
+  Context = Struct.new(:element, :feature, :prefix, :in_array, :in_block, :after_label, :problem)
 
   class << self
+  include RText::Tokenizer
 
   # Builds the context information based on a set of +content_lines+. Content lines
   # are the RText lines containing the nested command headers in the original order.
   # The cursor is assumed to be in the last context line at column +position_in_line+
   def build_context(language, context_lines, position_in_line)
-    context_info = fix_context(context_lines, position_in_line)
+    context_info = fix_context(language, context_lines, position_in_line)
     return nil unless context_info
     element = instantiate_context_element(language, context_info)
     if element
-      feature = context_info.role &&
-        element.class.ecore.eAllStructuralFeatures.find{|f| f.name == context_info.role}
-      Context.new(element, feature, context_info.prefix, context_info.in_array, context_info.in_block)
+      after_label = false
+      if context_info.role
+        if context_info.role.is_a?(Integer)
+          feature = language.unlabled_arguments(element.class.ecore)[context_info.role] 
+        else
+          feature = element.class.ecore.eAllStructuralFeatures.find{|f| f.name == context_info.role}
+          after_label = true
+        end
+      else
+        feature = nil
+      end
+      Context.new(element, feature, context_info.prefix, context_info.in_array, context_info.in_block, after_label, context_info.problem)
     else
-      Context.new(nil, nil, context_info.prefix, context_info.in_array, context_info.in_block)
+      Context.new(nil, nil, context_info.prefix, context_info.in_array, context_info.in_block, false, context_info.problem)
     end
   end
 
@@ -63,82 +74,124 @@ module ContextBuilder
   def find_leaf_child(element, num_required_children)
     childs = element.class.ecore.eAllReferences.select{|r| r.containment}.collect{|r|
       element.getGenericAsArray(r.name)}.flatten
-    if childs.size > 0
-      find_leaf_child(childs.first, num_required_children-1)
-    elsif num_required_children == 0
+    if num_required_children == 0
       element
+    elsif childs.size > 0
+      find_leaf_child(childs.first, num_required_children-1)
     else
       nil
     end
   end
 
-  ContextInternal = Struct.new(:lines, :num_elements, :role, :prefix, :in_array, :in_block)
+  ContextInternal = Struct.new(:lines, :num_elements, :role, :prefix, :in_array, :in_block, :problem)
 
   # extend +context_lines+ into a set of lines which can be processed by the RText
-  def fix_context(context_lines, position_in_line)
+  def fix_context(language, context_lines, position_in_line)
     context_lines = context_lines.dup
     # make sure there is at least one line
-    # the frontent may ommit the last context line if the cursor is at collumn 0
-    if context_lines.empty? || (position_in_line == 0 && context_lines.last != "")
+    if context_lines.empty?
       context_lines << ""
     end
     position_in_line ||= context_lines.last.size
     # cut off last line right of cursor
-    context_lines << context_lines.pop[0..position_in_line-1]
+    if position_in_line < 1
+      context_lines.pop
+      context_lines << ""
+    else
+      context_lines << context_lines.pop[0..position_in_line-1]
+    end
+    problem = nil
     line = context_lines.last
-    if line =~ /^\s*\w+\s+/
-      # this line contains a new element
-      num_elements = 1
-      in_block = false
-      # labled array value
-      if line =~ /\W(\w+):\s*\[([^\]]*)$/
-        role = $1
-        array_content = $2
-        in_array = true
-        if array_content =~ /,\s*(\S*)$/
-          prefix = $1
-          line.sub!(/,\s*\S*$/, "]")
-        else 
-          array_content =~ /\s*(\S*)$/
-          prefix = $1
-          line.sub!(/\[[^\]]*$/, "[]")
-        end
-      # labled value
-      elsif line =~ /\W(\w+):\s*(\S*)$/
-        role = $1
-        prefix = $2
+    if line =~ /\{\s*$/
+      # remove curly brace from last line, required for correct counting of num_elements
+      line.sub!(/\{\s*$/,"")
+      problem = :after_curly
+    end
+    
+    num_elements = in_block = in_array = missing_comma = role = prefix = nil
+    tokens = tokenize(line, language.reference_regexp)
+    tokens.pop if tokens.last && tokens.last.kind == :newline
+    if tokens.size > 0 && tokens[0].kind == :identifier
+      if tokens.size > 1 || line =~ /\s+$/
+        # this line contains a new element
+        num_elements = 1
+        in_block = false
         in_array = false
-        line.sub!(/\s*\w+:\s*\S*$/, "")
-        line.sub!(/,$/, "")
-      # unlabled value or label
-      elsif line =~ /[,\s](\S*)$/
         role = nil
-        prefix = $1
-        in_array = false
-        line.sub!(/\s*\S*$/, "")
-        line.sub!(/,$/, "")
-      # TODO: unlabled array value
-      else 
-        # parse problem
-        return nil
-      end
-    else
-      # this line is in the content block
-      num_elements = 0
-      in_block = true
-      # role or new element
-      if line =~ /^\s*(\w*)$/
-        prefix = $1
+        missing_comma = false
+        unlabled_index = 0
+        tokens[1..-1].each do |token|
+          if token.kind == "["
+            in_array = true
+          elsif token.kind == "]"
+            in_array = false
+            missing_comma = true
+            role = nil
+          elsif token.kind == :label
+            role = token.value.sub(/:$/, "")
+          elsif token.kind == ","
+            missing_comma = false
+            role = nil unless in_array
+            unlabled_index += 1 unless in_array
+          end
+        end
+        if ((tokens.size == 1 && line =~ /\s+$/) || 
+            tokens.last.kind == "," ||
+            in_array ||
+            ([:error, :string, :integer, :float, :boolean, :identifier, :reference].
+              include?(tokens.last.kind) && line !~ /\s$/)) &&
+            !tokens.any?{|t| t.kind == :label} &&
+            !(problem == :after_curly)
+          role ||= unlabled_index 
+        end
+        if [:string, :integer, :float, :boolean, :identifier, :reference].
+            include?(tokens.last.kind) && line =~ /\s$/ && tokens.size > 1
+            missing_comma = true
+            role = nil unless in_array
+        end
+        if [:error, :string, :integer, :float, :boolean, :identifier, :reference].
+            include?(tokens.last.kind) && line !~ /\s$/
+          prefix = tokens.last.value.to_s
+        else
+          prefix = ""
+        end
+      else
+        # in completion of command
+        num_elements = 0
+        missing_comma = false
+        in_block = (context_lines.size > 1)
+        prefix = tokens[0].value.to_s
         role, in_array = find_role(context_lines[0..-2])
         # fix single role lable
         if context_lines[-2] =~ /^\s*\w+:\s*$/
           context_lines[-1] = context_lines.pop
         end
-      else
-        # comment, closing brackets, etc.
-        return nil
       end
+    elsif line.strip.empty?
+      # in completion of command but without prefix
+      num_elements = 0
+      missing_comma = false
+      in_block = (context_lines.size > 1)
+      prefix = ""
+      role, in_array = find_role(context_lines[0..-2])
+      # fix single role lable
+      if context_lines[-2] =~ /^\s*\w+:\s*$/
+        context_lines[-1] = context_lines.pop
+      end
+    else
+      # comment, closing brackets, etc.
+      num_elements = 0
+      missing_comma = false
+      in_block = (context_lines.size > 1)
+      return nil
     end
+
+    # remove prefix, a value which is currently being completed should not be part of the
+    # context model
+    if prefix && prefix.size > 0
+      line.slice!(-(prefix.size)..-1)
+    end
+
     context_lines.reverse.each do |l|
       if l =~ /\{\s*$/
         context_lines << "}"
@@ -147,7 +200,8 @@ module ContextBuilder
         context_lines << "]"
       end
     end
-    ContextInternal.new(context_lines, num_elements, role, prefix, in_array, in_block)
+    problem = :missing_comma if !problem && missing_comma
+    ContextInternal.new(context_lines, num_elements, role, prefix, in_array, in_block, problem)
   end
 
   def find_role(context_lines)

data/lib/rtext/default_loader.rb

@@ -36,10 +36,13 @@ class DefaultLoader
   def initialize(language, fragmented_model, options={})
     @lang = language
     @model = fragmented_model
+    @files_added = []
+    @files_changed = []
+    @files_removed = []
     @change_detector = RGen::Util::FileChangeDetector.new(
-      :file_added => method(:file_added),
-      :file_removed => method(:file_removed),
-      :file_changed => method(:file_changed))
+      :file_added => lambda {|f| @files_added << f }, 
+      :file_removed => lambda {|f| @files_removed << f},
+      :file_changed => lambda {|f| @files_changed << f})
     @cache = options[:cache]
     @fragment_by_file = {}
     pattern = options[:pattern]
@@ -61,21 +64,75 @@ class DefaultLoader
   #    optionally, the proc may take second argument which is the overall number of files
   #    default: no after load proc
   #
+  #  :on_progress
+  #    a proc which is called when some progress is made 
+  #    receives the current fragment being loaded, the actual work done as an integer and
+  #    the overall work to be done as an integer
+  #    default: no on progress proc
+  #
   def load(options={})
     @before_load_proc = options[:before_load]
     @after_load_proc = options[:after_load]
     files = @file_provider.call 
     @num_files = files.size
+    @files_added = []
+    @files_changed = []
+    @files_removed = []
     @change_detector.check_files(files)
+    @progress_monitor = ProgressMonitor.new(options[:on_progress], @files_added + @files_changed)
+    @files_added.each {|f| file_added(f)}
+    @files_changed.each {|f| file_changed(f)}
+    @files_removed.each {|f| file_removed(f)}
+    @lang.reference_qualifier.call(@model.unresolved_refs, @model)
     @model.resolve(:fragment_provider => method(:fragment_provider),
       :use_target_type => @lang.per_type_identifier)
   end
 
   private
 
+  class ProgressMonitor
+    def initialize(on_progress_proc, files)
+      @on_progress_proc = on_progress_proc || lambda {|frag, work_done, work_overall| }
+      # there is a progress call twice for each element (in tokenizer and instantiator)
+      @work_overall = num_elements(files)*2
+      @work_done = 0
+      @work_last_sent = 0
+    end
+
+    def before_fragment_load(frag, kind)
+      @loading_cached = (kind == :load_cached)
+    end
+
+    def after_fragment_load(frag)
+      @work_done += frag.elements.size*2 if @loading_cached
+      @on_progress_proc.call(frag, @work_done, @work_overall)
+      @work_last_sent = @work_done
+    end
+
+    def instantiator_progress(frag)
+      @work_done += 1
+      if @work_done > @work_last_sent + 100
+        @on_progress_proc.call(frag, @work_done, @work_overall)
+        @work_last_sent = @work_done
+      end
+    end
+
+    private
+
+    def num_elements(files)
+      result = 0
+      files.each do |f|
+        content = File.open(f, "rb"){|fh| fh.read}
+        result += content.scan(/\n\s*\w+\s+/).size
+        result += 1 if content =~ /^\s*\w+\s+/
+      end
+      result
+    end
+  end
+
   def file_added(file)
     fragment = RGen::Fragment::ModelFragment.new(file, 
-      :identifier_provider => @lang.identifier_provider)
+      :identifier_provider => lambda {|e, c| @lang.identifier_provider.call(e, nil, nil, nil)})
     load_fragment_cached(fragment)
     @model.add_fragment(fragment)
     @fragment_by_file[file] = fragment
@@ -88,7 +145,7 @@ class DefaultLoader
 
   def file_changed(file)
     fragment = RGen::Fragment::ModelFragment.new(file, 
-      :identifier_provider => @lang.identifier_provider)
+      :identifier_provider => lambda {|e, c| @lang.identifier_provider.call(e, nil, nil, nil)})
     load_fragment_cached(fragment)
     if @dont_reload_with_errors && fragment.data[:problems].size > 0
       # keep old fragment but attach new problems
@@ -135,6 +192,7 @@ class DefaultLoader
   end
 
   def call_before_load_proc(fragment, kind)
+    @progress_monitor.before_fragment_load(fragment, kind)
     if @before_load_proc
       if @before_load_proc.arity == 3
         @before_load_proc.call(fragment, kind, @num_files) 
@@ -145,6 +203,7 @@ class DefaultLoader
   end
 
   def call_after_load_proc(fragment)
+    @progress_monitor.after_fragment_load(fragment)
     if @after_load_proc
       if @after_load_proc.arity == 2
         @after_load_proc.call(fragment, @num_files) 
@@ -160,20 +219,26 @@ class DefaultLoader
     problems = []
     root_elements = []
     inst = RText::Instantiator.new(@lang)
-    File.open(fragment.location) do |f|
+    File.open(fragment.location, "rb") do |f|
       inst.instantiate(f.read,
         :env => env,
         :unresolved_refs => urefs,
         :problems => problems,
         :root_elements => root_elements,
         :fragment_ref => fragment.fragment_ref,
-        :file_name => fragment.location)
+        :file_name => fragment.location,
+        :on_progress => lambda do 
+          @progress_monitor.instantiator_progress(fragment)
+        end)
     end
-    fragment.data = {:problems => problems}
+    # data might have been created during instantiation (e.g. comment or annotation handler)
+    fragment.data ||= {}
+    fragment.data[:problems] = problems
     fragment.set_root_elements(root_elements,
       :unresolved_refs => urefs, 
       :elements => env.elements)
     fragment.build_index
+    @lang.reference_qualifier.call(urefs, fragment)
     fragment.resolve_local(:use_target_type => @lang.per_type_identifier)
   end