modl 0.3.26 → 0.3.27

data/lib/modl/parser/parser.rb

@@ -1,72 +1,285 @@
 # frozen_string_literal: true
 
-# The MIT License (MIT)
-#
-# Copyright (c) 2019 NUM Technology Ltd
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-
-require 'modl/parser/throwing_error_listener'
-require 'modl/parser/parsed'
-
 module MODL
+  # A Parser module
   module Parser
+    REPLACEMENTS = {
+      '\t' => "\t",
+      '\n' => "\n",
+      '\b' => "\b",
+      '\f' => "\f",
+      '\r' => "\r",
+      '~t' => "\t",
+      '~n' => "\n",
+      '~b' => "\b",
+      '~f' => "\f",
+      '~r' => "\r",
+      '~\\' => '\\',
+      '\\\\' => '\\',
+      '~~' => '~',
+      '\~' => '~',
+      '~(' => '(',
+      '\(' => '(',
+      '~)' => ')',
+      '\)' => ')',
+      '~[' => '[',
+      '\[' => '[',
+      '~]' => ']',
+      '\]' => ']',
+      '~;' => ';',
+      '\;' => ';',
+      '~`' => '`',
+      '\`' => '`',
+      '~"' => '"',
+      '"' => '"',
+      '~=' => '=',
+      '\=' => '='
+    }.freeze
+
+    def self.parse_modl(str)
+      tokens = MODL::Tokeniser.tokenise str
+      if tokens.nil? || tokens.empty?
+        MODL::Model::Modl.new nil
+      elsif root_primitive? tokens[0]
+        MODL::Model::Modl.new tokens[0].value
+      else
+        result = parse_structures(tokens)
+        return MODL::Model::Modl.new result[0] if result.length == 1
+
+        MODL::Model::Modl.new ModlArray.new(result)
+      end
+    end
+
+    def self.root_primitive?(token)
+      %i[string quoted null true false integer float].include?(token)
+    end
+
+    def self.parse_structures(tokens)
+      result = []
+      until tokens.empty?
+        result.push parse_modl_value(tokens)
+        expect_separator = tokens.shift
+        if !expect_separator.nil? && expect_separator.type == :struct_sep
+          raise ParserError, "Expected ';' near #{tokens}"
+        end
+      end
+      result
+    end
+
+    def self.parse_modl_value(tokens)
+      first_token = tokens.shift
+
+      case first_token.type
+      when :lbracket
+        tokens.unshift first_token
+        return parse_modl_array tokens
+      when :lparen
+        tokens.unshift first_token
+        return parse_modl_map tokens
+      when :string || :quoted
+        peek = tokens[0]
+        key = first_token.value
+        if !peek.nil? && peek.type == :equals
+          tokens.shift
+          return MODL::Model::ModlPair.new replace_escapes(unquote(key)), parse_pair_value(tokens)
+        end
+
+        if !peek.nil? && (peek.type == :lbracket || peek.type == :lparen)
+          return MODL::Model::ModlPair.new replace_escapes(unquote(key)), parse_pair_value(tokens)
+        end
+
+        if peek.nil? || peek.type == :struct_sep || peek.type == :rparen || peek.type == :rbracket
+          return MODL::Model::ModlString.new(replace_escapes(first_token.value)) if first_token.type == :string
+
+          return MODL::Model::ModlQuoted.new(replace_escapes(unquote(first_token.value)))
+        end
+      when :integer
+        return MODL::Model::ModlInteger.new first_token.value
+      when :float
+        return MODL::Model::ModlFloat.new first_token.value
+      when :null
+        return MODL::Model::ModlBoolNull::MODL_NULL
+      when true
+        return MODL::Model::ModlBoolNull::MODL_TRUE
+      when false
+        return MODL::Model::ModlBoolNull::MODL_FALSE
+      else
+        tokens.unshift first_token
+        maybe_primitive = parse_primitive tokens
+        return maybe_primitive unless maybe_primitive.nil?
+      end
+      raise ParserError, "Unexpected token: \'#{first_token}\'"
+    end
+
+    def self.parse_pair_value(tokens)
+      first_token = tokens.shift
+      case first_token.type
+      when :lbracket
+        tokens.unshift first_token
+        return parse_modl_array tokens
+      when :lparen
+        tokens.unshift first_token
+        return parse_modl_map tokens
+      when :string || :quoted
+        peek = tokens[0]
+
+        raise ParserError, "Unexpected token: \'#{first_token}\'" if !peek.nil? && peek.type == :equals
+
+        if !peek.nil? && (peek.type == :lbracket || peek.type == :lparen)
+          raise ParserError, "Unexpected token: \'#{first_token}\'"
+        end
+
+        if peek.nil? || peek.type == :struct_sep || peek.type == :rparen || peek.type == :rbracket
+          return MODL::Model::ModlString.new replace_escapes(first_token.value) if first_token.type == :string
+
+          return MODL::Model::ModlQuoted.new replace_escapes(unquote(first_token.value))
+        end
+        raise ParserError, "Unexpected token: \'#{first_token}\'"
+      when :integer
+        return MODL::Model::ModlInteger.new first_token.value
+      when :float
+        return MODL::Model::ModlFloat.new first_token.value
+      when :null
+        return MODL::Model::ModlBoolNull::MODL_NULL
+      when true
+        return MODL::Model::ModlBoolNull::MODL_TRUE
+      when false
+        return MODL::Model::ModlBoolNull::MODL_FALSE
+      else
+        tokens.unshift first_token
+        maybe_primitive = parse_primitive tokens
+        return maybe_primitive unless maybe_primitive.nil?
+      end
+      raise ParserError, "Unexpected token: \'#{first_token}\'"
+    end
 
-    # This class converts the input string into a Modl:Parser::Parsed object for further processing.
-    class Parser
-      def self.parse(str, global = nil)
-        begin
-          lexer = MODL::Parser::MODLLexer.new(Antlr4::Runtime::CharStreams.from_string(str, 'String'))
-          lexer.remove_error_listeners
-          lexer.add_error_listener ThrowingErrorListener.instance
-
-          tokens = Antlr4::Runtime::CommonTokenStream.new(lexer)
-
-          parser = MODL::Parser::MODLParser.new(tokens)
-          parser.remove_error_listeners
-          parser.add_error_listener ThrowingErrorListener.instance
-
-          global = GlobalParseContext.new if global.nil?
-
-          parsed = Parsed.new(global)
-          parser.modl.enter_rule(parsed)
-          parsed
-        rescue Antlr4::Runtime::ParseCancellationException => e
-          check_modl_version(global, e)
-          raise ParserError, 'Parser Error: ' + e.message
-        rescue StandardError => e
-          check_modl_version(global, e)
-          raise InterpreterError, 'Interpreter Error: ' + e.message
-        rescue InterpreterError => e
-          check_modl_version(global, e)
-          raise InterpreterError, 'Interpreter Error: ' + e.message
+    def self.parse_primitive(tokens)
+      result = nil
+      tok = tokens.shift
+
+      case tok.type
+      when :lparen || :rparen || :lbracket || :rbracket || :equals
+        raise ParserError, "Unexpected token: \'#{tok}\'" if tokens.empty?
+
+        tokens.unshift tok
+        return nil
+      when :null
+        result = MODL::Model::ModlBoolNull::MODL_NULL
+      when true
+        result = MODL::Model::ModlBoolNull::MODL_TRUE
+      when false
+        result = MODL::Model::ModlBoolNull::MODL_FALSE
+      when :quoted
+        result = MODL::Model::ModlQuoted.new replace_escapes(unquote(tok.value))
+      when :string
+        result = MODL::Model::ModlString.new replace_escapes(tok.value)
+      when :integer
+        result = MODL::Model::ModlInteger.new tok.value
+      when :float
+        result = MODL::Model::ModlFloat.new tok.value
+      else
+        raise ParserError, "Unknown token type in: \'#{tok}\'"
+      end
+
+      peek = tokens[0]
+      raise ParserError, 'Only one primitive allowed at the root.' if !peek.nil? && peek.type == :struct_sep
+
+      if !peek.nil? && (peek.type == :lparen || peek.type == :lbracket || peek.type == :equals)
+        s.unshift tok
+        return nil
+      elsif !peek.nil?
+        raise ParserError, "Unexpected token: \'#{peek}\'"
+      end
+      result
+    end
+
+    def self.parse_modl_map(tokens)
+      first_token = tokens.shift
+      entries = []
+
+      until tokens.empty?
+        peek = tokens[0]
+        if !peek.nil? && peek.type == :rparen
+          tokens.shift
+          break
+        end
+
+        mp = parse_modl_value tokens
+        entries.push mp
+
+        peek = tokens[0]
+
+        raise ParserError, "Expected ')' near #{first_token}" if peek.nil?
+
+        case peek.type
+        when :rparen
+          tokens.shift
+          break
+        when :struct_sep
+          tokens.shift
+          peek = tokens[0]
+          raise ParserError, "Unexpected ; before ] at #{peek}" if !peek.nil? && peek.type == :rparen
+        end
+      end
+      MODL::Model::ModlMap.new entries
+    end
+
+    def self.parse_modl_array(tokens)
+      first_token = tokens.shift
+      entries = []
+
+      until tokens.empty?
+        peek = tokens[0]
+        if !peek.nil? && peek.type == :rbracket
+          tokens.shift
+          break
+        end
+
+        mp = parse_modl_value tokens
+        entries.push mp
+
+        peek = tokens[0]
+
+        raise ParserError, "Expected ']' near #{first_token}" if peek.nil?
+
+        case peek.type
+        when :rbracket
+          tokens.shift
+          break
+        when :struct_sep
+          tokens.shift
+          peek = tokens[0]
+          raise ParserError, "Unexpected ; before ] at #{peek}" if !peek.nil? && peek.type == :rparen
         end
       end
+      MODL::Model::ModlArray.new entries
+    end
+
+    def self.unquote(str)
+      return str unless str.instance_of?(String)
+
+      if (str.start_with?('`') && str.end_with?('`')) || (str.start_with?('"') && str.end_with?('"'))
+        str[1..-2]
+      else
+        str
+      end
+    end
+
+    def self.replace_escapes(str)
+      return str unless str.instance_of?(String)
 
-      def self.check_modl_version(global, e)
-        if global.syntax_version > global.interpreter_syntax_version
-          raise InterpreterError, 'Interpreter Error: ' + e.message + ' - MODL Version ' +
-              global.interpreter_syntax_version.to_s +
-              ' interpreter cannot process this MODL Version ' +
-              global.syntax_version.to_s + ' file.'
+      result = str
+      i = 0
+      while i < str.length
+        REPLACEMENTS.each_pair do |key, value|
+          if result.slice(i..).start_with?(key)
+            result = result.sub(key, value)
+            break
+          end
         end
+        i += 1
       end
+      result
     end
   end
 end

data/lib/modl/tokeniser/context.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module MODL
+  module Tokeniser
+    # A parsing context for a MODL String
+    class Context
+      WS = " \t\r\n"
+      INTEGER_REGEX = '^-?(?:0|[1-9]\d*)$'
+      FLOAT_REGEX = '^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$'
+      NON_STRING_TOKENS = '[]();"=`'
+
+      def initialize(str)
+        @str = str
+        @tokens = []
+        @tok_start = 0
+      end
+
+      def parse
+        while next_token
+        end
+        @tokens
+      end
+
+      def next_token
+        while @tok_start < @str.length
+          break unless WS.include?(@str[@tok_start])
+
+          @tok_start += 1
+        end
+
+        return false if @tok_start >= @str.length
+
+        case @str[@tok_start]
+        when '('
+          tok_type = :lparen
+          tok_end = @tok_start + 1
+        when ')'
+          tok_type = :rparen
+          tok_end = @tok_start + 1
+        when '['
+          tok_type = :lbracket
+          tok_end = @tok_start + 1
+        when ']'
+          tok_type = :rbracket
+          tok_end = @tok_start + 1
+        when ';'
+          tok_type = :struct_sep
+          tok_end = @tok_start + 1
+        when '='
+          tok_type = :equals
+          tok_end = @tok_start + 1
+        when '"'
+          tok_type = :quoted
+          tok_end = scan_to_end_of_quoted(@str, @tok_start, '"')
+        when '`'
+          tok_type = :quoted
+          tok_end = scan_to_end_of_quoted(@str, @tok_start, '`')
+        else
+          tok_type = :string
+          tok_end = scan_to_end_of_string(@str, @tok_start)
+        end
+
+        tok_value = @str[@tok_start..(tok_end - 1)].strip
+
+        if tok_value.match? INTEGER_REGEX
+          number = tok_value.to_i
+          @tokens.push Token.new(:integer, number, @tok_start, tok_end)
+        elsif tok_value.match? FLOAT_REGEX
+          number = tok_value.to_f
+          @tokens.push Token.new(:float, number, @tok_start, tok_end)
+        elsif tok_value == 'null'
+          @tokens.push Token.new(:null, nil, @tok_start, tok_end)
+        elsif tok_value == 'true'
+          @tokens.push Token.new(true, true, @tok_start, tok_end)
+        elsif tok_value == 'false'
+          @tokens.push Token.new(false, false, @tok_start, tok_end)
+        else
+          @tokens.push Token.new(tok_type, tok_value, @tok_start, tok_end)
+        end
+
+        @tok_start = tok_end
+        tok_end < @str.length
+      end
+
+      def scan_to_end_of_quoted(str, start, quote_char)
+        end_str = start + 1
+        while end_str < str.length
+          end_char = str[end_str]
+          prev_char = str[end_str - 1]
+
+          break if end_char == quote_char && prev_char != '\\' && prev_char != '~'
+
+          end_str += 1
+        end
+        end_str + 1
+      end
+
+      def scan_to_end_of_string(str, start)
+        end_str = start + 1
+        while end_str < str.length
+          break if NON_STRING_TOKENS.include?(str[end_str]) && !escaped?(str, end_str - 1)
+
+          end_str += 1
+        end
+        end_str
+      end
+
+      def escaped?(str, pos)
+        pos >= 0 && (str[pos] == '~' || str[pos] == '\\')
+      end
+    end
+  end
+end

data/lib/modl/tokeniser/tokeniser.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'modl/tokeniser/context'
+
+module MODL
+  # A MODL Tokeniser module
+  module Tokeniser
+    # A class to represent MODL tokens
+    class Token
+      attr_reader :type, :value, :from, :to
+
+      def initialize(type, value, from, to)
+        @type = type
+        @value = value
+        @from = from
+        @to = to
+      end
+
+      def to_s
+        "type: #{@type}, from: #{@from}, to: #{@to}, value: \"#{@value}\""
+      end
+    end
+
+    def self.tokenise(str)
+      Context.new(str).parse
+    end
+  end
+end

data/lib/modl/util/functions.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module MODL
+  # Helper functions for MODL.generate
+  module UTIL
+    SHOULD_BE_GRAVE_QUOTED = /.*[()\[\];{}="].*/.freeze
+    IS_NUMERIC = /^-?[0-9]*\.?[0-9]+(?:[Ee][+-]?[0-9]+)?$/.freeze
+
+    def self.escape_graves(str)
+      return str unless str
+
+      str.gsub(/`/, '~`')
+    end
+
+    def self.escape_double_quotes(str)
+      return str unless str
+
+      i = str.index('"', 1) # ignore the initial quote
+      result = str
+      # Don't affect the final quote in the string
+      while !i.nil? && i < (result.length - 1)
+        result = result.slice(0..(i - 1)) + '~u0022' + result.slice((i + 1)..)
+        i = result.index('"', 1)
+      end
+      result
+    end
+
+    def self.grave_quote_if_necessary(str)
+      return str unless str
+
+      if str.match?(SHOULD_BE_GRAVE_QUOTED) ||
+         (str.match?(IS_NUMERIC) && str != '00' && str != '01' && str != '000') ||
+         (str.strip.empty? ||
+          str.match?(IS_NUMERIC) ||
+          str == 'true' ||
+          str == 'false' ||
+          str == 'null')
+        "`#{str}`"
+      else
+        str
+      end
+    end
+
+    def self.double_quote_if_necessary(str)
+      if str && str.include?('~`')
+        "\"#{str}\""
+      else
+        str
+      end
+    end
+
+    def self.replace_nbsp(str)
+      str.gsub('\u00a0', ' ')
+    end
+
+    def self.escape_and_quote(str)
+      double_quote_if_necessary(
+        grave_quote_if_necessary(
+          escape_graves(
+            UNICODE.escape(
+              replace_nbsp(
+                escape_double_quotes(str)
+              )
+            )
+          )
+        )
+      )
+    end
+
+    def self.non_string_primitive?(str)
+      str == 'true' || str == 'false' || str == 'null' || str.match?(IS_NUMERIC)
+    end
+  end
+end

data/lib/modl/util/unicode.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: false
+
+module MODL
+  # Handle escaping of unicode characters.
+  module UNICODE
+    VERTICAL_TAB = 0x0b
+    BACKSPACE = 0x08
+    FORMFEED = 0x0c
+    LINEFEED = 0x0a
+    CARRIAGE_RETURN = 0x0d
+    TAB = 0x09
+
+    def self.escape_char(chr)
+      return '' unless chr
+
+      if chr >= 32 && chr <= 127
+        '' << chr
+      elsif chr == VERTICAL_TAB
+        '~u000B'
+      elsif chr == BACKSPACE
+        '\\b'
+      elsif chr == FORMFEED
+        '\\f'
+      elsif chr == LINEFEED
+        '\\n'
+      elsif chr == CARRIAGE_RETURN
+        '\\r'
+      elsif chr == TAB
+        '\\t'
+      else
+        '' << chr
+      end
+    end
+
+    def self.escape(str)
+      result = ''
+
+      str.each_codepoint do |c|
+        result << escape_char(c)
+      end
+      result
+    end
+  end
+end

data/lib/modl/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module MODL
+  VERSION = '0.3.27'
+end

data/lib/modl.rb

@@ -1,34 +1,9 @@
 # frozen_string_literal: true
 
-# The MIT License (MIT)
-#
-# Copyright (c) 2019 NUM Technology Ltd
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-
-require "antlr4/runtime"
-require "modl/parser/version"
-require 'modl/parser/MODLParserListener'
-require 'modl/parser/MODLParserVisitor'
-require 'modl/parser/MODLParserBaseListener'
-require 'modl/parser/MODLParserBaseVisitor'
-require 'modl/parser/MODLLexer'
-require 'modl/parser/MODLParser'
-require 'modl/parser/interpreter'
-require 'modl/parser/instruction_processor'
+require 'modl/version'
+require 'modl/model/model'
+require 'modl/tokeniser/tokeniser'
+require 'modl/parser/parser'
+require 'modl/util/unicode'
+require 'modl/util/functions'
+require 'modl/interpreter'

data/modl.gemspec

@@ -1,30 +1,27 @@
-
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'modl/parser/version'
+require 'modl/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'modl'
-  spec.version       = MODL::Parser::VERSION
+  spec.version       = MODL::VERSION
   spec.authors       = ['Tony Walmsley']
   spec.email         = ['tony@aosd.co.uk']
 
-  spec.summary       = 'The ANTLR4 Lexer and Parser for MODL generated using the Ruby language target.'
-  spec.description   = 'Contains the base Lexer, Parser, and supporting classes for a MODL parser.'
-  spec.homepage      = 'https://github.com/MODLanguage/ruby-interpreter'
+  spec.summary       = 'A Ruby inretpreter for the MODL data serialisation language.'
+  spec.description   = 'A command line program for interpreting MODL objects.'
+  spec.homepage      = 'https://github.com/MODLanguage/ruby-modl'
   spec.license       = 'MIT'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency "rake", ">= 12.3.3"
+  spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_runtime_dependency 'antlr4-runtime', '= 0.2.10'
-  spec.add_runtime_dependency 'punycode4r', '>= 0.2.0'
 end