modl 0.3.10 → 0.3.11

data/lib/modl/parser/MODLParser.tokens

@@ -12,30 +12,29 @@ LSBRAC=11
 RSBRAC=12
 NUMBER=13
 COMMENT=14
-STRING=15
-HASH_PREFIX=16
-QUOTED=17
-GRAVED=18
-LCBRAC=19
-CWS=20
-QMARK=21
-FSLASH=22
-GTHAN=23
-LTHAN=24
-ASTERISK=25
-AMP=26
-PIPE=27
-EXCLAM=28
-CCOMMENT=29
-RCBRAC=30
+QUOTED=15
+STRING=16
+HASH_PREFIX=17
+LCBRAC=18
+CWS=19
+QMARK=20
+FSLASH=21
+GTHAN=22
+LTHAN=23
+ASTERISK=24
+AMP=25
+PIPE=26
+EXCLAM=27
+CCOMMENT=28
+RCBRAC=29
 ','=8
-'{'=19
-'?'=21
-'/'=22
-'>'=23
-'<'=24
-'*'=25
-'&'=26
-'|'=27
-'!'=28
-'}'=30
+'{'=18
+'?'=20
+'/'=21
+'>'=22
+'<'=23
+'*'=24
+'&'=25
+'|'=26
+'!'=27
+'}'=29

data/lib/modl/parser/evaluator.rb

@@ -54,7 +54,7 @@ module MODL
         while i < condition.values.length
           item = condition.values[i]
           if item.primitive.constant
-            value2 = item.text
+            value2 = Substitutions.process(item.text)
           else
             value2, success = value(global, item.text)
           end
@@ -105,11 +105,12 @@ module MODL
           if ikey.to_s == key
             index_val = global.index[ikey]
             value1 = index_val.respond_to?(:text) ? index_val.text : nil
+            value1 = Substitutions.process(value1)
           else
             pair = global.pair(key)
-            return k unless pair
+            return Substitutions.process(k) unless pair
 
-            value1 = pair.text
+            value1 = Substitutions.process(pair.text)
           end
           success = true
         end

data/lib/modl/parser/file_importer.rb

@@ -70,8 +70,8 @@ module MODL
               else
                 txt = File.readlines(file_name).join
               end
-            rescue
-              raise InterpreterError, 'File not found: ' + file_name
+            rescue StandardError => e
+              raise InterpreterError, 'File not found: ' + file_name + ', error: ' + e.message
             end
             global.loaded_file(file_name)
 

data/lib/modl/parser/modl_index.rb

@@ -33,7 +33,7 @@ module MODL
         # collect all values from the object
         if item.is_a? Parsed::ParsedValueItem
           if item&.value&.text
-            global.index << item.value.text
+            global.add_to_index(item.value.text)
           elsif item&.value&.array
             item.value.array.abstractArrayItems.each do |avi|
               global.add_to_index(avi.arrayValueItem)

data/lib/modl/parser/modl_method.rb

@@ -40,18 +40,18 @@ module MODL
         transform = @transform
         while transform && transform.length > 0
           if transform.start_with? 'replace'
-            close_bracket = transform.index(')')
+            close_bracket = transform.index('>')
             m = Sutil.head(transform, close_bracket + 1).sub!('replace', 'r')
             str = StandardMethods.run_method(m, str)
             # Consume the subst clause
-            close_bracket = transform.index(')')
+            close_bracket = transform.index('>')
             transform = Sutil.tail(transform, close_bracket + 2)
           elsif transform.start_with? 'trim'
-            close_bracket = transform.index(')')
+            close_bracket = transform.index('>')
             m = Sutil.head(transform, close_bracket + 1).sub!('trim', 't')
             str = StandardMethods.run_method(m, str)
             # Consume the trunc clause
-            close_bracket = transform.index(')')
+            close_bracket = transform.index('>')
             transform = Sutil.tail(transform, close_bracket + 2)
           elsif transform.start_with? 'initcap'
             str = str.split.map(&:capitalize) * ' '
@@ -129,7 +129,7 @@ module MODL
           CGI.escape(str)
         when 'r', 'replace'
           s1, s2 = get_subst_parts(mthd)
-          str.sub(s1, s2)
+          str.gsub(s1, s2)
         when 't', 'trim'
           s1 = extract_params mthd
           i = str.index(s1)
@@ -153,11 +153,15 @@ module MODL
       # Extract the method parameter
       def self.extract_params(str)
         # should be of the form .r(s1,s2)
-        Sutil.between(str, '(', ')')
+        Sutil.between(str, '<', '>')
       end
 
       def self.valid_method?(mthd)
-        return @@mthd_names.include?(mthd)
+        m = mthd
+        if m.include?('<')
+          m = Sutil.until(m, '<')
+        end
+        return @@mthd_names.include?(m)
       end
     end
   end

data/lib/modl/parser/parsed.rb

@@ -202,12 +202,15 @@ module MODL
         attr_accessor :type # A string set to the type of pair that we have found bases on its key
         attr_accessor :text # The simple text value rather than the object
         attr_accessor :final
+        attr_accessor :loaded # For *load instructions so we don't load twice
+
 
         def initialize(global)
           @global = global
           @needs_defref = true
           @final = false
           @file_importer = FileImporter.instance
+          @loaded = false
         end
 
         def find_property(key)
@@ -271,11 +274,7 @@ module MODL
           value = @valueItem.extract_hash if @valueItem
           value = @map.extract_hash if @map
 
-          if value.is_a?(String) && (value.include?('%') || value.start_with?('`'))
-            @text, _ignore = RefProcessor.deref(@text, @global)
-          else
-            @text = value
-          end
+          @text = value
 
           return if @type == 'index'
           return if @type == 'hidden'
@@ -294,8 +293,6 @@ module MODL
 
           ctx_string = ctx.STRING
           @key = ctx_string.to_s unless ctx_string.nil?
-          ctx_string = ctx.NUMBER
-          @key = ctx_string.to_s unless ctx_string.nil?
           ctx_quoted = ctx.QUOTED
           unless ctx_quoted.nil?
             @key = ctx_quoted.to_s
@@ -365,7 +362,7 @@ module MODL
           when 'import'
             files = @valueItem.extract_hash if @valueItem
             files = @array.extract_hash if @array
-            @file_importer.import_files files, @global
+            @file_importer.import_files files, @global unless @global.in_condition?
           when 'index'
             IndexExtractor.extract(self, @global)
           when 'hidden'
@@ -664,7 +661,7 @@ module MODL
             if user_method
               return user_method.run(@string.string)
             end
-            return StandardMethods.run_method(key, @string.string)
+            return StandardMethods.run_method(key, Substitutions.process(@string.string))
           end
         end
 
@@ -707,13 +704,11 @@ module MODL
             @text = ctx_string.text
 
             @constant = @text.start_with?('`') && !@text.include?('%') && !@text.include?('`.')
-            @text = Parsed.additional_string_processing(@text)
             @string = ParsedString.new(@text)
             @text = @string.string
           elsif !ctx_quoted.nil?
             @constant = true
-            @text = Sutil.toptail(ctx_quoted.text) # remove the quotes
-            @text = Parsed.additional_string_processing(@text)
+            @text = ctx_quoted.text
             @quoted = ParsedQuoted.new(@text)
           elsif !ctx_null.nil?
             @nilVal = ParsedNull.instance
@@ -1023,6 +1018,7 @@ module MODL
           @global = global
           @topLevelConditionalReturns = []
           @conditionTests = []
+          @file_importer = FileImporter.instance
         end
 
         def extract_hash
@@ -1033,7 +1029,17 @@ module MODL
             if item.structures[0].pair
               key = item.structures[0].pair.key
               key = Sutil.tail(key) if key[0] == '_'
-              @global.pair(key, item.structures[0].pair)
+              pair = item.structures[0].pair
+              @global.pair(key, pair)
+
+              if key.downcase.start_with? '*l'
+                files = pair.valueItem.extract_hash if pair.valueItem
+                files = pair.array.extract_hash if pair.array
+                unless pair.loaded
+                  @file_importer.import_files files, @global
+                  pair.loaded = true
+                end
+              end
             end
             return item.extract_hash
           end
@@ -1043,7 +1049,17 @@ module MODL
           if last_item.structures[0].pair
             key = last_item.structures[0].pair.key
             key = Sutil.tail(key) if key[0] == '_'
-            @global.pair(key, last_item.structures[0].pair)
+            pair = last_item.structures[0].pair
+            @global.pair(key, pair)
+
+            if key.downcase.start_with? '*l'
+              files = pair.valueItem.extract_hash if pair.valueItem
+              files = pair.array.extract_hash if pair.array
+              unless pair.loaded
+                @file_importer.import_files files, @global
+                pair.loaded = true
+              end
+            end
           end
           last_item.extract_hash
         end

data/lib/modl/parser/ref_processor.rb

@@ -32,18 +32,19 @@ module MODL
     class RefProcessor
 
       NESTED_SEPARATOR = '.'
-      MATCHER = Regexp.new('((`?\%[0-9][0-9.][a-zA-Z0-9.(),]*`?)|(`?\%[0-9][0-9]*`?)|(`?\%[_a-zA-Z][_a-zA-Z0-9.%(),]*`?)|(`.*`\.[_a-zA-Z0-9.(),%]+)|(`.*`))')
+      MATCHER = Regexp.new('((%\w+)(\.\w*<`?\w*`?,`\w*`>)+|(%` ?[\w-]+`[\w.<>,]*%?)|(%\*?[\w]+(\.%?\w*<?[\w,]*>?)*%?))')
+      MAX_RECURSE_DEPTH = 10
 
       def self.trivial_reject(str)
         # do a fast check to see if we need to deref - save processing the regex if we don't have to.
-        str.is_a?(String) && (str.nil? || str.include?('%') || str.include?('`'))
+        !(str.is_a?(String) && !str.start_with?('%*') && (str.nil? || str.include?('%') || str.include?('`')))
       end
 
       # Check str for references and process them.
       # Return the processed string and a new_value if there is one.
       def self.deref(str, global)
         obj = str
-        obj, new_value = split_by_ref_tokens str, global if trivial_reject(str)
+        obj, new_value = split_by_ref_tokens str, global unless trivial_reject(str)
         [obj, new_value]
       end
 
@@ -80,9 +81,9 @@ module MODL
 
 
           ref = match[0]
-          ref = Sutil.head(ref) if ref.end_with?('%')
           text = Sutil.after(text, ref)
-          new_value, remainder = expand(global, ref)
+
+          new_value, remainder = expand(0, global, ref)
           ref = Sutil.until(ref, remainder)
           if new_value.is_a?(String)
             str = str.sub(ref, new_value)
@@ -101,7 +102,7 @@ module MODL
               str = if ref == str
                       new_value.text
                     else
-                      str.sub(ref, new_value.text.to_s)
+                      str.sub(ref, Sutil.unquote(new_value.text.to_s))
                     end
               new_value = nil
             else
@@ -115,7 +116,10 @@ module MODL
         return new_value, str
       end
 
-      def self.expand(global, ref)
+      def self.expand(depth, global, ref)
+        if depth > MAX_RECURSE_DEPTH
+          raise InterpreterError, 'Recursing too deep to resolve: "' + ref + '"'
+        end
         result = nil
         prev = nil
 
@@ -123,12 +127,13 @@ module MODL
 
         parts = Sutil.tail(degraved).split('.') if degraved[0] == '%'
         parts = degraved.split('.') unless degraved[0] == '%'
+        parts[-1] = Sutil.head(parts[-1]) if parts[-1].end_with?('%')
 
         if degraved.include?('%')
           resolved = 0
           parts.each do |p|
             if p.include?('%')
-              p, _ignore = expand(global, p)
+              p, _ignore = expand(depth + 1, global, p)
               if p.is_a?(MODL::Parser::MODLParserBaseListener)
                 p = p.text
               end
@@ -154,7 +159,8 @@ module MODL
                            end
                          end
                        elsif result.is_a? Parsed::ParsedPair
-                         if result.text
+                         prop = result.find_property(p)
+                         if result.text && !prop
                            if StandardMethods.valid_method?(p)
                              StandardMethods.run_method(p, result.text)
                            else
@@ -166,7 +172,27 @@ module MODL
                              end
                            end
                          else
-                           result.find_property(p)
+                           prop
+                         end
+                       elsif result.is_a? Parsed::ParsedArrayValueItem
+                         prop = result.find_property(p)
+                         if result.text && !prop
+                           if StandardMethods.valid_method?(p)
+                             result_text = result.text
+                             if result_text.start_with?('`') && result_text.end_with?('`')
+                               result_text = Sutil.toptail(result_text)
+                             end
+                             StandardMethods.run_method(p, result_text)
+                           else
+                             mthd = global.user_method(p)
+                             if !mthd.nil?
+                               mthd.run(result.text)
+                             else
+                               mthd
+                             end
+                           end
+                         else
+                           prop
                          end
                        elsif result.is_a? Array
                          nil
@@ -175,7 +201,14 @@ module MODL
                            raise InterpreterError, 'Interpreter Error: Invalid object reference: ' + degraved
                          end
                          if result.nil?
-                           global.pair(p)
+                           unless ref.start_with?('%`')
+                             a_pair = global.pair(p)
+                           end
+                           if a_pair.nil?
+                             p
+                           else
+                             a_pair
+                           end
                          else
                            result.find_property(p)
                          end
@@ -192,6 +225,9 @@ module MODL
           else
             remainder = resolved < parts.length ? '.' + parts[resolved..parts.length].join('.') : ''
           end
+          if (prev == Sutil.between(ref, '%', '%')) || (ref.start_with?('%') && prev == Sutil.tail(ref))
+            prev = ref
+          end
           [prev, remainder]
         else
           # Remove the graves if there are any.

data/lib/modl/parser/substitutions.rb

@@ -76,14 +76,19 @@ module MODL
       # Replace all escape sequences in the supplied string and return the new value.
       def self.process(str)
         return str unless str.is_a? String
+
+        # Remove unescaped graves and double quotes
+        new_str = Sutil.unquote(str)
+
+        # Handle escape sequences
         @@subs.each do |s|
           loop do
-            prev = str
-            str = str.sub(s[0], s[1])
-            break unless str && str != prev
+            prev = new_str
+            new_str = new_str.sub(s[0], s[1])
+            break unless new_str && new_str != prev
           end
         end
-        str
+        new_str
       end
     end
   end

data/lib/modl/parser/sutil.rb

@@ -97,4 +97,12 @@ module Sutil
     end
     result
   end
+
+  def self.unquote(str)
+    new_str = str
+    if (str.start_with?('`') && str.end_with?('`')) || (str.start_with?('"') && str.end_with?('"'))
+      new_str = Sutil.toptail(str)
+    end
+    new_str
+  end
 end

data/lib/modl/parser/version.rb

@@ -24,6 +24,6 @@
 
 module MODL
   module Parser
-    VERSION = "0.3.10"
+    VERSION = "0.3.11"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: modl
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.3.11
 platform: ruby
 authors:
 - Tony Walmsley
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-07 00:00:00.000000000 Z
+date: 2019-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: antlr4-runtime
@@ -109,6 +109,8 @@ files:
 - grammar_tests/c.modl
 - grammar_tests/demo_config.modl
 - grammar_tests/error_tests.json
+- grammar_tests/files/a-b.txt
+- grammar_tests/files/c-d.txt
 - grammar_tests/import_config.modl
 - grammar_tests/test_import_dir/nested_import1.txt
 - grammar_tests/test_import_dir/nested_import2.txt