modl 0.3.6 → 0.3.7

data/grammar_tests/error_tests.json

@@ -66,5 +66,12 @@
   "*V=test",
   "*VERSION=0",
   "*VERSION=0.1",
-  "a=b;*VERSION=1"
+  "a=b;*VERSION=1",
+  "*class(*id=a;*name=alpha;*superclass=str);a=null",
+  "*class(*id=a;*name=alpha;*superclass=num);a=null",
+  "*class(*id=a;*name=alpha;*superclass=num);a=x",
+  "*class(*id=a;*name=alpha;*superclass=num);a=null",
+  "*class(*id=a;*name=alpha;*superclass=map);a=[1;2;3]",
+  "*class(*id=a;*name=alpha;*superclass=arr);a=(c=d)",
+  "*L=grammar_tests/test_import_dir/nested_import1.txt;files=%*load"
 ]

data/lib/modl/parser/MODLParser.rb

@@ -307,7 +307,7 @@ class MODLParser < Antlr4::Runtime::Parser
 				@_state_number = 69
 				modl_pair()
 			else
-				raise Antlr4::Runtime::NoViableAltException self
+				raise Antlr4::Runtime::NoViableAltException, self
 			end
 		rescue Antlr4::Runtime::RecognitionException => re
 			_localctx.exception = re
@@ -621,7 +621,7 @@ class MODLParser < Antlr4::Runtime::Parser
 							match(COLON)
 
 						else
-							raise Antlr4::Runtime::NoViableAltException self
+							raise Antlr4::Runtime::NoViableAltException, self
 						end
 						@_state_number = 118 
 						@_err_handler.sync(self)
@@ -630,7 +630,7 @@ class MODLParser < Antlr4::Runtime::Parser
 					end
 
 				else
-					raise Antlr4::Runtime::NoViableAltException self
+					raise Antlr4::Runtime::NoViableAltException, self
 				end
 				@_state_number = 122 
 				@_err_handler.sync(self)
@@ -1240,7 +1240,7 @@ class MODLParser < Antlr4::Runtime::Parser
 				@_state_number = 192
 				modl_map_conditional()
 			else
-				raise Antlr4::Runtime::NoViableAltException self
+				raise Antlr4::Runtime::NoViableAltException, self
 			end
 		rescue Antlr4::Runtime::RecognitionException => re
 			_localctx.exception = re
@@ -1476,7 +1476,7 @@ class MODLParser < Antlr4::Runtime::Parser
 				@_state_number = 218
 				modl_array_conditional()
 			else
-				raise Antlr4::Runtime::NoViableAltException self
+				raise Antlr4::Runtime::NoViableAltException, self
 			end
 		rescue Antlr4::Runtime::RecognitionException => re
 			_localctx.exception = re

data/lib/modl/parser/class_processor.rb

@@ -4,7 +4,7 @@ module MODL
     # It works recursively since class usage can be nested.
     class ClassProcessor
       # How deep can the class structure be?
-      MAX_RECURSION_DEPTH = 20
+      MAX_RECURSION_DEPTH = 50
       # global is a GlobalParseContext and obj is the extracted Array or Hash from MODL::Parser::Parsed.extract_json
       def self.process(global, obj)
         # Process each object in the array or just process the object if its a hash.
@@ -22,29 +22,8 @@ module MODL
 
       private
 
-      def self.check_class_type(global, key, value)
-        clazz = global.classs(key)
-        top = top_class(clazz, global)
-        case top
-        when 'num'
-          unless value.is_a?(Numeric)
-            raise InterpreterError, 'Interpreter Error: Numeric value expected, but found: ' + value.to_s + ' of type ' + value.class.to_s
-          end
-          return value
-        when 'str'
-          return value.to_s
-        end
-        value
-      end
-
       # Process the contents of the supplied hash obj
       def self.process_obj(global, obj)
-        if obj.length == 1
-          k = obj.keys[0]
-          nv = check_class_type(global, k, obj[k])
-          obj[k] = nv
-        end
-
         obj.keys.each do |k|
           value = obj[k]
           # Does the key refer to a class that we have parsed or loaded?
@@ -76,7 +55,32 @@ module MODL
         end
       end
 
-      # Convert the supplied object val into an instance of the class with key k
+      def self.has_assign_statement?(clazz, global, depth = 0)
+        # Check for *assign statements
+        return if depth > MAX_RECURSION_DEPTH
+        return nil? if clazz.nil?
+        return true unless clazz.assign.nil?
+
+        superclass = clazz.superclass
+        c = global.classs(superclass)
+        return has_assign_statement?(c, global, depth + 1) if c
+
+        false
+      end
+
+      def self.has_inherited_pairs?(clazz, global, depth = 0)
+        # Check for *assign statements
+        return if depth > MAX_RECURSION_DEPTH
+        return nil? if clazz.nil?
+        return true unless clazz.content.empty?
+
+        superclass = clazz.superclass
+        c = global.classs(superclass)
+        return has_inherited_pairs?(c, global, depth + 1) if c
+
+        false
+      end
+
       def self.process_class(global, k, v)
         clazz = global.classs(k)
         if k != clazz.id && !(v.is_a?(Hash) || v.is_a?(Array))
@@ -99,22 +103,78 @@ module MODL
         elsif v.is_a?(String)
           # Safe to ignore
         else
-          # Check the top class and do some type-specific processing
-          tc = top_class(clazz, global)
-          if tc == 'str'
-            new_value = v.to_s
-          elsif tc == 'num' && !v.is_a?(Numeric)
-            raise InterpreterError, 'Superclass of "' + clazz.id + '" is num - cannot assign String value "' + v.to_s + '"'
-          elsif tc == 'map'
-            if v.is_a? Hash
-              # Bring down values from the superclass hierarchy
-              new_value = copy_from_superclasses(clazz, global, new_value, v)
+          # Safe to ignore
+        end
+
+        # Check the top class and do some type-specific processing
+        tc = top_class(clazz, global)
+        if tc.nil?
+          # There is no defined top class so we need to infer it base on the value
+          # and the rules defined here: https://github.com/MODLanguage/grammar/wiki/Class-Supertype-Processing
+          #
+          if has_assign_statement?(clazz, global)
+            tc = 'map'
+          else
+            if has_inherited_pairs?(clazz, global)
+              tc = 'map'
             else
-              new_value[k] = v
+              if v.is_a? String
+                tc = 'str'
+              elsif v.is_a? Numeric
+                tc = 'num'
+              elsif (v.is_a? TrueClass) || (v.is_a? FalseClass)
+                tc = 'bool'
+              elsif v.nil?
+                tc = 'null'
+              elsif v.is_a? Array
+                tc = 'arr'
+              elsif v.is_a? Hash
+                tc = 'map'
+              end
             end
+          end
+        end
+        if tc == 'str'
+          raise InterpreterError, "Interpreter Error: Cannot convert null value to string." if v.nil?
+          new_value = v.to_s
+        elsif tc == 'num'
+          if (v.is_a? String) && (v.to_i.to_s == v.to_s)
+            new_value = v.to_i
+          elsif v.is_a? TrueClass
+            new_value = 1
+          elsif v.is_a? FalseClass
+            new_value = 0
+          elsif v.is_a? Numeric
+            new_value = v
           else
+            raise InterpreterError, 'Superclass of "' + clazz.id + '" is num - cannot assign value "' + v.to_s + '"'
+          end
+        elsif tc == 'bool'
+          new_value = v
+        elsif tc == 'null'
+          new_value = nil
+        elsif tc == 'arr'
+          if v.is_a? Array
             new_value = v
+          elsif v.is_a? Hash
+            raise InterpreterError, 'Interpreter Error: Cannot convert map to array: ' + v.to_s
+          else
+            new_value = [v]
+          end
+        elsif tc == 'map'
+          if new_value.is_a? Hash
+            # Bring down values from the superclass hierarchy
+            new_value = copy_from_superclasses(clazz, global, new_value, v)
+          elsif v.is_a? Array
+            raise InterpreterError, 'Interpreter Error: Cannot convert array to map: ' + v.to_s
+          else
+            new_value = {}
+            new_value['value'] = v
+            # Bring down values from the superclass hierarchy
+            new_value = copy_from_superclasses(clazz, global, new_value, v)
           end
+        elsif tc.nil? && (v.is_a? Hash)
+          new_value = v
         end
 
         [clazz.name_or_id, new_value]
@@ -122,7 +182,12 @@ module MODL
 
       # Bring down values from the superclass hierarchy
       def self.copy_from_superclasses(clazz, global, new_value, v)
-        new_value = v.merge(new_value)
+        if v.is_a? Hash
+          new_value = v.merge(new_value)
+        end
+
+        clazz.merge_content(new_value)
+
         depth = 0
         loop do
           clazz = global.classs(clazz.superclass)

data/lib/modl/parser/evaluator.rb

@@ -91,73 +91,6 @@ module MODL
         end
         [value1, success]
       end
-
-      def evaluate_old
-        result = false
-        if @key
-          if @key.include?('%')
-            value1, _ignore = MODL::Parser::RefProcessor.deref(@key, @global)
-          else
-            key = @key
-            ikey = key.to_i
-            if ikey.to_s == key
-              index_val = @global.index[ikey]
-              value1 = index_val.respond_to?(:text) ? index_val.text : nil
-            else
-              pair = @global.pair(key)
-              return false unless pair
-
-              value1 = pair.text
-            end
-          end
-
-          @values.each do |value|
-            value2 = value.text
-            value2, _ignore = MODL::Parser::RefProcessor.deref(value2, @global) if value2.is_a?(String) && value2.include?('%')
-            value2 = @global.pair(value.text).text if @global.pair(value.text)
-
-            case @operator
-            when '='
-              wild = value2.is_a?(String) && value2.include?('*') ? true : false
-              if wild
-                regex = '^'
-                value2.each_char do |c|
-                  regex << (c == '*' ? '.*' : c)
-                end
-                result |= !value1.match(regex).nil?
-              else
-                result |= value1 == value2
-              end
-            when '>'
-              result |= value1 > value2
-            when '<'
-              result |= value1 < value2
-            when '>='
-              result |= value1 >= value2
-            when '<='
-              result |= value1 <= value2
-            end
-            break if result # shortcut if we have a matching value
-          end
-        elsif @values.length == 1
-          key = @values[0].text
-          if key.is_a?(String)
-            key = key.start_with?('%') ? Sutil.tail(key) : key
-          end
-          the_pair = @global.pair(key)
-          if the_pair
-            result = the_pair.text
-          else
-            return true if @values[0].trueVal
-            return false if @values[0].falseVal
-            return false if @values[0].string
-
-            result = @values[0].evaluate
-          end
-        end
-        result
-      end
-
     end
   end
 end

data/lib/modl/parser/file_importer.rb

@@ -9,7 +9,7 @@ module MODL
     class FileImporter
       include Singleton
 
-      CACHE_DISABLED = true
+      CACHE_DISABLED = false
 
       def initialize
         @cache = ObjectCache.new
@@ -36,31 +36,36 @@ module MODL
           end
 
           # Did we hit the cache?
-          if (parsed.nil? && CACHE_DISABLED) || (CACHE_DISABLED)
+          if parsed.nil? || CACHE_DISABLED
             # No.
 
             begin
-              uri = URI(file_name)
-              txt = Net::HTTP.get(uri)
-            rescue
-              begin
+              if file_name.start_with?('http')
+                uri = URI(file_name)
+                txt = Net::HTTP.get(uri)
+              else
                 txt = File.readlines(file_name).join
-              rescue
-                raise InterpreterError, 'File not found: ' + file_name
               end
+            rescue
+              raise InterpreterError, 'File not found: ' + file_name
             end
-
             global.loaded_file(file_name)
 
             # Parse the downloaded file ands extract the classes
-            parsed = MODL::Parser::Parser.parse txt, global
+            new_parse_context = GlobalParseContext.new
+            new_parse_context.merge_pairs(global)
+            parsed = MODL::Parser::Parser.parse txt, new_parse_context
             # Save it for next time
             @cache.put(file_name, parsed) unless CACHE_DISABLED
+          else
+            global.loaded_file(file_name)
           end
           # Extract the JSON content and add the classes and pairs to the existing GlobalParseContext hashes.
           parsed.extract_hash
           global.merge_classes(parsed.global)
           global.merge_pairs(parsed.global)
+          global.merge_loaded_files(parsed.global)
+          global.structures.concat(parsed.structures)
         end
       end
     end

data/lib/modl/parser/global_parse_context.rb

@@ -6,9 +6,16 @@ module MODL
     class GlobalParseContext
 
       attr_accessor :syntax_version
+      attr_accessor :max_files_allowed
+      attr_accessor :structures
       attr_reader :interpreter_syntax_version
+      attr_reader :loaded_files
 
       def initialize
+        # This keeps track of all loaded structures from parsed files.
+        @structures = []
+        # This is set to true if we encounter a *L[OAD] instruction
+        @max_files_allowed = 2147483647
         # Holds the index array from a MODL file, e.g. '?=a:b:c:d'
         @index = []
         # Contains all pairs as they are encountered in the parsing process.
@@ -29,6 +36,8 @@ module MODL
 
       def loaded_file(str)
         @loaded_files << str unless str.nil?
+        raise InterpreterError, 'Cannot load multiple files after *LOAD instruction' if @loaded_files.length > @max_files_allowed
+
       end
 
       def index_value(n, default)
@@ -41,6 +50,10 @@ module MODL
         @conditional.positive?
       end
 
+      def freeze_max_files(plus)
+        @max_files_allowed = @loaded_files.length + plus
+      end
+
       def enter_condition
         @conditional += 1
       end
@@ -75,6 +88,11 @@ module MODL
         @classes_by_name.merge!(other.all_classes_by_name)
       end
 
+      def merge_loaded_files(other)
+        @loaded_files.concat(other.loaded_files)
+        raise InterpreterError, 'Cannot load multiple files after *LOAD instruction' if @loaded_files.length > @max_files_allowed
+      end
+
       def has_pairs?
         @pairs.length.positive?
       end

data/lib/modl/parser/modl_method.rb

@@ -127,6 +127,10 @@ module MODL
         # should be of the form .r(s1,s2)
         Sutil.between(str, '(', ')')
       end
+
+      def self.valid_method?(mthd)
+        return 'udisertp'.include?(mthd)
+      end
     end
   end
 end

data/lib/modl/parser/parsed.rb

@@ -40,11 +40,12 @@ module MODL
           @structures << structure
         end
 
+        @structures = global.structures + structures
+
         @global
       end
 
       def self.additional_string_processing(text)
-        text = Substitutions.process text
         # Special case for a possibly empty graved string ``
         unless text.nil?
           match_data = /^`([^`]*)`$/.match text
@@ -246,7 +247,7 @@ module MODL
           value = @valueItem.extract_hash if @valueItem
           value = @map.extract_hash if @map
 
-          if value.is_a?(String) && (value.start_with?('%') || value.start_with?('`'))
+          if value.is_a?(String) && (value.include?('%') || value.start_with?('`'))
             @text, _ignore = RefProcessor.deref(@text, @global)
           else
             @text = value
@@ -277,6 +278,8 @@ module MODL
             @key = Sutil.toptail(@key) # remove the quotes
           end
 
+          raise InterpreterError, 'Invalid key - null, true, or false keys are not allowed.' if @key.nil?
+
           if @key.include?('%') || @key.include?('`')
             @key, new_value = RefProcessor.deref @key, @global
             unless @key.is_a?(String)
@@ -287,10 +290,6 @@ module MODL
 
           @final = true if @key.upcase == @key
 
-          set_pair_type
-
-          raise InterpreterError, 'Invalid keyword: ' + @key if @type == 'pair' && @key.start_with?('*')
-
           modl_array = ctx.modl_array
           modl_map = ctx.modl_map
           modl_value_item = ctx.modl_value_item
@@ -306,6 +305,9 @@ module MODL
             modl_value_item.enter_rule(@valueItem)
           end
 
+          set_pair_type
+          raise InterpreterError, 'Invalid keyword: ' + @key if @type == 'pair' && @key.start_with?('*')
+
           validate_key if @type == 'pair' || @type == 'hidden'
 
           # Type-specific processing
@@ -377,7 +379,7 @@ module MODL
         end
 
         def validate_key
-          invalid_chars = "!$@-+'*#^&"
+          invalid_chars = "£!$@-+'*#^&"
           invalid_chars.each_char do |c|
             next unless @key.include?(c)
 
@@ -435,6 +437,15 @@ module MODL
           if @key == '*L' || @key == '*LOAD'
             @key = @key.downcase
             @type = 'import'
+            if @array
+              @global.freeze_max_files(@array.abstractArrayItems.length)
+            elsif @valueItem&.value&.array
+              @global.freeze_max_files(@valueItem&.value&.array.abstractArrayItems.length)
+            elsif @valueItem&.value&.nbArray
+              @global.freeze_max_files(@valueItem&.value&.nbArray.arrayItems.length)
+            else
+              @global.freeze_max_files(1)
+            end
           end
           if @key == '*l' || @key == '*load'
             @type = 'import'
@@ -636,7 +647,7 @@ module MODL
         def extract_hash
           result, _ignore = RefProcessor.deref(@text, @global) unless @constant
           result = @text if @constant
-          result
+          Substitutions.process result
         end
 
         def evaluate
@@ -670,6 +681,7 @@ module MODL
             @text = @number.num
           elsif !ctx_string.nil?
             @text = ctx_string.text
+
             @constant = @text.start_with?('`') && !@text.include?('%') && !@text.include?('`.')
             @text = Parsed.additional_string_processing(@text)
             @string = ParsedString.new(@text)
@@ -706,7 +718,7 @@ module MODL
         end
 
         def extract_hash
-          @string
+          @string = Substitutions.process @string
         end
       end
 
@@ -914,8 +926,7 @@ module MODL
         end
 
         def extract_hash
-          result = @conditionTests[0].evaluate
-          return @mapConditionalReturns[0].extract_hash if result
+          return @mapConditionalReturns[0].extract_hash if @result
 
           @mapConditionalReturns[1].extract_hash if @mapConditionalReturns.length > 1
         end
@@ -942,6 +953,7 @@ module MODL
             end
             i += 1
           end
+          @result = @conditionTests[0].evaluate
         end
       end
 
@@ -1147,10 +1159,8 @@ module MODL
         end
 
         def extract_hash
-          result = @conditionTests[0].evaluate
-
-          return result if @valueConditionalReturns.length == 0
-          return @valueConditionalReturns[0].extract_hash if result
+          return @result if @valueConditionalReturns.length == 0
+          return @valueConditionalReturns[0].extract_hash if @result
           return @valueConditionalReturns[1].extract_hash
         end
 
@@ -1165,7 +1175,7 @@ module MODL
 
             @conditionTests[i] = condition_test
 
-            return if ctx.modl_value_conditional_return_i(i).nil?
+            break if ctx.modl_value_conditional_return_i(i).nil?
 
             conditional_return = ParsedValueConditionalReturn.new @global
 
@@ -1183,6 +1193,7 @@ module MODL
 
             i += 1
           end
+          @result = @conditionTests[0].evaluate
         end
       end
 

data/lib/modl/parser/ref_processor.rb

@@ -42,10 +42,23 @@ module MODL
       def self.process_tokens(global, original, str, text)
         new_value = nil
         loop do
-          match = MATCHER.match(text.to_s)
+          text_s = text.to_s
+          match = MATCHER.match(text_s)
           break if match.nil?
 
+          match_index = text_s.index(match[0])
+          if match_index > 0
+            if text_s[match_index - 1] == '~' || text_s[match_index - 1] == '\\'
+              break
+            end
+            if text_s[match_index + match.length] == '~' || text_s[match_index + match.length] == '\\'
+              break
+            end
+          end
+
+
           ref = match[0]
+          ref = Sutil.head(ref) if ref.end_with?('%')
           text = Sutil.after(text, ref)
           new_value, remainder = expand(global, ref)
           ref = Sutil.until(ref, remainder)
@@ -59,6 +72,8 @@ module MODL
                     str.sub(ref, nv_text.to_s)
                   end
             new_value = nil
+          elsif new_value.is_a?(Parsed::ParsedMapItem)
+            raise InterpreterError, 'Interpreter Error: Found a map when expecting an array'
           elsif new_value.is_a?(MODL::Parser::MODLParserBaseListener)
             if new_value.text
               str = if ref == str
@@ -99,6 +114,9 @@ module MODL
             n = p.to_i
             result = if n.to_s == p
                        # Numeric ref
+                       if !result.nil? && !result.respond_to?(:find_property)
+                         raise InterpreterError, 'Interpreter Error: Invalid obejct reference: ' + degraved
+                       end
                        result.nil? ? global.index_value(n, degraved) : result.find_property(n)
                      else
                        # String ref
@@ -107,6 +125,9 @@ module MODL
                        elsif result.is_a? Array
                          nil
                        else
+                         if !result.nil? && !result.respond_to?(:find_property)
+                           raise InterpreterError, 'Interpreter Error: Invalid obejct reference: ' + degraved
+                         end
                          result.nil? ? global.pair(p) : result.find_property(p)
                        end
                      end
@@ -126,8 +147,16 @@ module MODL
           # Remove the graves if there are any.
           result = parts[0]
           i = 1
+          stalled = false
           while i < parts.length
-            result = StandardMethods.run_method(parts[i], result)
+            stalled |= StandardMethods.valid_method?(parts[i]) ? false : true
+
+            if stalled
+              result << '.'
+              result << parts[i]
+            else
+              result = StandardMethods.run_method(parts[i], result)
+            end
             i += 1
           end
           [result, '']

data/lib/modl/parser/substitutions.rb

@@ -53,6 +53,7 @@ module MODL
 
       # Replace all escape sequences in the supplied string and return the new value.
       def self.process(str)
+        return str unless str.is_a? String
         @@subs.each do |s|
           loop do
             prev = str

data/lib/modl/parser/version.rb

@@ -1,5 +1,5 @@
 module MODL
   module Parser
-    VERSION = "0.3.6"
+    VERSION = "0.3.7"
   end
 end

data/modl.gemspec

@@ -26,6 +26,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'antlr4-runtime', '>= 0.0.1'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_runtime_dependency 'antlr4-runtime', '= 0.2.3'
+  spec.add_runtime_dependency 'antlr4-runtime', '= 0.2.6'
   spec.add_runtime_dependency 'punycode4r', '>= 0.2.0'
 end