nestedtext 3.2.1 → 4.2.0

data/lib/nestedtext/parser.rb

@@ -1,18 +1,21 @@
 # frozen_string_literal: true
 
-require "stringio"
+require 'stringio'
 
-require "nestedtext/errors_internal"
-require "nestedtext/scanners"
-require "nestedtext/constants"
+require 'nestedtext/errors_internal'
+require 'nestedtext/scanners'
+require 'nestedtext/constants'
+require 'nestedtext/inline_parser'
 
 module NestedText
   # A LL(1) recursive descent parser for NT.
-  class Parser
+  class Parser # rubocop:disable Metrics/ClassLength
     def self.assert_valid_top_level_type(top_class)
-      unless !top_class.nil? && top_class.is_a?(Class) && TOP_LEVEL_TYPES.map(&:object_id).include?(top_class.object_id)
-        raise Errors::UnsupportedTopLevelTypeError, top_class
+      if !top_class.nil? && top_class.is_a?(Class) && TOP_LEVEL_TYPES.map(&:object_id).include?(top_class.object_id)
+        return
       end
+
+      raise Errors::UnsupportedTopLevelTypeError, top_class
     end
 
     def initialize(io, top_class, strict: false)
@@ -21,42 +24,64 @@ module NestedText
       @top_class = top_class
       @strict = strict
       @line_scanner = LineScanner.new(io)
-      @inline_scanner = nil
     end
 
     def parse
       result = parse_any(0)
       case @top_class.object_id
       when Object.object_id
-        raise Errors::AssertionError, "Parsed result is of unexpected type." if
-        !result.nil? && ![Hash, Array, String].include?(result.class) && @strict
+        return_object(result)
       when Hash.object_id
-        result = {} if result.nil?
-        raise Errors::TopLevelTypeMismatchParsedTypeError.new(@top_class, result) unless result.instance_of?(Hash)
+        return_hash(result)
       when Array.object_id
-        result = [] if result.nil?
-        raise Errors::TopLevelTypeMismatchParsedTypeError.new(@top_class, result) unless result.instance_of?(Array)
+        return_array(result)
       when String.object_id
-        result = "" if result.nil?
-        raise Errors::TopLevelTypeMismatchParsedTypeError.new(@top_class, result) unless result.instance_of?(String)
+        return_string(result)
       else
         raise Errors::UnsupportedTopLevelTypeError, @top_class
       end
-      result
     end
 
     private
 
+    def return_object(result)
+      raise Errors::AssertionError, 'Parsed result is of unexpected type.' if \
+        !result.nil? && ![Hash, Array, String].include?(result.class) && @strict
+
+      result
+    end
+
+    def return_hash(result)
+      result = {} if result.nil?
+      raise Errors::TopLevelTypeMismatchParsedTypeError.new(@top_class, result) unless result.instance_of?(Hash)
+
+      result
+    end
+
+    def return_array(result)
+      result = [] if result.nil?
+      raise Errors::TopLevelTypeMismatchParsedTypeError.new(@top_class, result) unless result.instance_of?(Array)
+
+      result
+    end
+
+    def return_string(result)
+      result = '' if result.nil?
+      raise Errors::TopLevelTypeMismatchParsedTypeError.new(@top_class, result) unless result.instance_of?(String)
+
+      result
+    end
+
     def assert_valid_input_type(input)
-      unless input.nil? || input.is_a?(IO) || input.is_a?(StringIO)
-        raise Errors::WrongInputTypeError.new([IO, StringIO], input)
-      end
+      return if input.nil? || input.is_a?(IO) || input.is_a?(StringIO)
+
+      raise Errors::WrongInputTypeError.new([IO, StringIO], input)
     end
 
     def parse_any(indentation)
       return nil if @line_scanner.peek.nil?
 
-      case @line_scanner.peek.tag # TODO: Use Null Pattern instead with a EndOfInput tag?
+      case @line_scanner.peek.tag
       when :list_item
         parse_list_item(indentation)
       when :dict_item, :key_item
@@ -74,27 +99,101 @@ module NestedText
       end
     end
 
+    def parse_list_item_value(indentation, value)
+      return value unless value.nil?
+
+      if !@line_scanner.peek.nil? && @line_scanner.peek.indentation > indentation
+        parse_any(@line_scanner.peek.indentation)
+      elsif @line_scanner.peek.nil? || @line_scanner.peek.tag == :list_item
+        ''
+      end
+    end
+
+    def assert_list_line(line, indentation)
+      Errors.raise_unrecognized_line(line) if line.tag == :unrecognized
+      raise Errors::ParseLineTypeExpectedListItemError, line unless line.tag == :list_item
+      raise Errors::ParseInvalidIndentationError.new(line, indentation) if line.indentation != indentation
+    end
+
     def parse_list_item(indentation)
       result = []
       while !@line_scanner.peek.nil? && @line_scanner.peek.indentation >= indentation
         line = @line_scanner.read_next
+        assert_list_line(line, indentation)
+        result << parse_list_item_value(indentation, line.attribs['value'])
+      end
+      result
+    end
 
-        Errors.raise_unrecognized_line(line) if line.tag == :unrecognized
-        raise Errors::ParseLineTypeExpectedListItemError, line unless line.tag == :list_item
-        raise Errors::ParseInvalidIndentationError.new(line, indentation) if line.indentation != indentation
-
-        value = line.attribs["value"]
-        if value.nil?
-          if !@line_scanner.peek.nil? && @line_scanner.peek.indentation > indentation
-            value = parse_any(@line_scanner.peek.indentation)
-          elsif @line_scanner.peek.nil? || @line_scanner.peek.tag == :list_item
-            value = ""
-          end
+    def deserialize_custom_class(hash, first_line)
+      return hash unless !@strict && hash.length == 2 && hash.key?(CUSTOM_CLASS_KEY)
+
+      class_name = hash[CUSTOM_CLASS_KEY]
+      begin
+        clazz = class_name == 'nil' ? NilClass : Object.const_get(class_name, false)
+      rescue NameError
+        raise Errors::ParseCustomClassNotFoundError.new(first_line, class_name)
+      end
+      raise Errors::ParseCustomClassNoCreateMethodError.new(first_line, class_name) unless clazz.respond_to? :nt_create
+
+      clazz.nt_create(hash['data'])
+    end
+
+    def parse_kv_dict_item(indentation, line)
+      key = line.attribs['key']
+      value = line.attribs['value']
+      if value.nil?
+        value = ''
+        if !@line_scanner.peek.nil? && @line_scanner.peek.indentation > indentation
+          value = parse_any(@line_scanner.peek.indentation)
         end
+      end
+      [key, value]
+    end
 
-        result << value
+    def parse_key_item_key(indentation, line)
+      key = line.attribs['key']
+      while @line_scanner.peek&.tag == :key_item && @line_scanner.peek.indentation == indentation
+        line = @line_scanner.read_next
+        key += "\n#{line.attribs['key']}"
       end
-      result
+      key
+    end
+
+    def parse_key_item_value(indentation, line)
+      return '' if @line_scanner.peek.nil?
+
+      exp_types = %i[dict_item key_item list_item string_item]
+      unless exp_types.member?(@line_scanner.peek.tag)
+        raise Errors::ParseLineTypeNotExpectedError.new(line, exp_types, line.tag)
+      end
+
+      unless @line_scanner.peek.indentation > indentation
+        raise Errors::ParseMultilineKeyNoValueError,
+              line
+      end
+
+      parse_any(@line_scanner.peek.indentation)
+    end
+
+    def parse_kv_key_item(indentation, line)
+      key = parse_key_item_key(indentation, line)
+      value = parse_key_item_value(indentation, line)
+      [key, value]
+    end
+
+    def parse_dict_key_value(line, indentation)
+      if line.tag == :dict_item
+        parse_kv_dict_item(indentation, line)
+      else
+        parse_kv_key_item(indentation, line)
+      end
+    end
+
+    def assert_dict_item_line(line, indentation)
+      Errors.raise_unrecognized_line(line) if line.tag == :unrecognized
+      raise Errors::ParseInvalidIndentationError.new(line, indentation) if line.indentation != indentation
+      raise Errors::ParseLineTypeExpectedDictItemError, line unless %i[dict_item key_item].include? line.tag
     end
 
     def parse_dict_item(indentation)
@@ -103,170 +202,35 @@ module NestedText
       while !@line_scanner.peek.nil? && @line_scanner.peek.indentation >= indentation
         line = @line_scanner.read_next
         first_line = line if first_line.nil?
-        Errors.raise_unrecognized_line(line) if line.tag == :unrecognized
-        raise Errors::ParseInvalidIndentationError.new(line, indentation) if line.indentation != indentation
-        raise Errors::ParseLineTypeExpectedDictItemError, line unless %i[dict_item key_item].include? line.tag
-
-        value = nil
-        key = nil
-        if line.tag == :dict_item
-          key = line.attribs["key"]
-          value = line.attribs["value"]
-          if value.nil?
-            value = ""
-            if !@line_scanner.peek.nil? && @line_scanner.peek.indentation > indentation
-              value = parse_any(@line_scanner.peek.indentation)
-            end
-          end
-        else # :key_item
-          key = line.attribs["key"]
-          while @line_scanner.peek&.tag == :key_item && @line_scanner.peek.indentation == indentation
-            line = @line_scanner.read_next
-            key += "\n" + line.attribs["key"]
-          end
-          exp_types = %i[dict_item key_item list_item string_item]
-          if @line_scanner.peek.nil?
-            value = ""
-          else
-            unless exp_types.member?(@line_scanner.peek.tag)
-              raise Errors::ParseLineTypeNotExpectedError.new(line, exp_types, line.tag)
-            end
-            raise Errors::ParseMultilineKeyNoValueError, line unless @line_scanner.peek.indentation > indentation
-
-            value = parse_any(@line_scanner.peek.indentation)
-          end
-        end
+        assert_dict_item_line(line, indentation)
+        key, value = parse_dict_key_value(line, indentation)
         raise Errors::ParseDictDuplicateKeyError, line if result.key? key
 
         result[key] = value
       end
 
-      # Custom class decoding.
-      if !@strict && result.length == 2 && result.key?(CUSTOM_CLASS_KEY)
-        class_name = result[CUSTOM_CLASS_KEY]
-        begin
-          clazz = class_name == "nil" ? NilClass : Object.const_get(class_name, false)
-        rescue NameError
-          raise Errors::ParseCustomClassNotFoundError.new(first_line, class_name)
-        end
-        if clazz.respond_to? :nt_create
-          result = clazz.nt_create(result["data"])
-        else
-          raise Errors::ParseCustomClassNoCreateMethodError.new(first_line, class_name)
-        end
-      end
+      deserialize_custom_class(result, first_line)
+    end
 
-      result
+    def assert_string_line(line, indentation)
+      raise Errors::ParseInvalidIndentationError.new(line, indentation) if line.indentation != indentation
+      raise Errors::ParseLineTypeNotExpectedError.new(line, %i[string_item], line.tag) unless line.tag == :string_item
     end
 
     def parse_string_item(indentation)
       result = []
       while !@line_scanner.peek.nil? && @line_scanner.peek.indentation >= indentation
         line = @line_scanner.read_next
-        raise Errors::ParseInvalidIndentationError.new(line, indentation) if line.indentation != indentation
-        raise Errors::ParseLineTypeNotExpectedError.new(line, %i[string_item], line.tag) unless line.tag == :string_item
+        assert_string_line(line, indentation)
 
-        value = line.attribs["value"]
+        value = line.attribs['value']
         result << value
       end
       result.join("\n")
     end
 
-    def parse_inline_key
-      key = []
-      until @inline_scanner.empty? || [":", "{", "}", "[", "]", ","].include?(@inline_scanner.peek)
-        key << @inline_scanner.read_next
-      end
-      if @inline_scanner.empty?
-        raise Errors::ParseInlineNoClosingDelimiterError.new(@inline_scanner.line,
-                                                             @inline_scanner.pos)
-      end
-
-      last_char = @inline_scanner.read_next
-      if last_char == "}" && key.empty?
-        raise Errors::ParseInlineMissingValueError.new(@inline_scanner.line, @inline_scanner.pos - 1)
-      end
-      unless last_char == ":"
-        raise Errors::ParseInlineDictKeySyntaxError.new(@inline_scanner.line, @inline_scanner.pos - 1, last_char)
-      end
-
-      key.join.strip
-    end
-
-    def parse_inline
-      return nil if @inline_scanner.peek.nil?
-
-      result = nil
-      # Trim leading whitespaces
-      @inline_scanner.read_next while !@inline_scanner.empty? && [" ", "\t"].include?(@inline_scanner.peek)
-      case @inline_scanner.peek
-      when "{"
-        result = {}
-        first = true
-        loop do
-          @inline_scanner.read_next
-          break if first && @inline_scanner.peek == "}"
-
-          first = false
-          key = parse_inline_key
-          value = parse_inline
-          result[key] = value
-          break unless @inline_scanner.peek == ","
-        end
-        if @inline_scanner.empty?
-          raise Errors::ParseInlineNoClosingDelimiterError.new(@inline_scanner.line,
-                                                               @inline_scanner.pos)
-        end
-        last_char = @inline_scanner.read_next
-        unless last_char == "}"
-          raise Errors::ParseInlineDictSyntaxError.new(@inline_scanner.line, @inline_scanner.pos - 1,
-                                                       last_char)
-        end
-
-      when "["
-        result = []
-        first = true # TODO: can be replaced by checking result.empty? below?
-        loop do
-          @inline_scanner.read_next
-          break if first && @inline_scanner.peek == "]"
-
-          first = false
-          result << parse_inline
-          break unless @inline_scanner.peek == ","
-        end
-        if @inline_scanner.empty?
-          raise Errors::ParseInlineNoClosingDelimiterError.new(@inline_scanner.line,
-                                                               @inline_scanner.pos)
-        end
-        last_char = @inline_scanner.read_next
-
-        if last_char != "]"
-          if result[-1] == ""
-            raise Errors::ParseInlineMissingValueError.new(@inline_scanner.line, @inline_scanner.pos - 1)
-          else
-            raise Errors::ParseInlineListSyntaxError.new(@inline_scanner.line, @inline_scanner.pos - 1,
-                                                         last_char)
-          end
-        end
-      else # Inline string
-        inline_string = []
-        until @inline_scanner.empty? || ["{", "}", "[", "]", ","].include?(@inline_scanner.peek)
-          inline_string << @inline_scanner.read_next
-        end
-        result = inline_string.join.rstrip # Trim trailing whitespaces that lead up to next break point.
-      end
-      # Trim trailing whitespaces
-      @inline_scanner.read_next while !@inline_scanner.empty? && [" ", "\t"].include?(@inline_scanner.peek)
-      result
-    end
-
     def parse_inline_dict
-      @inline_scanner = InlineScanner.new(@line_scanner.read_next)
-      result = parse_inline
-      unless @inline_scanner.empty?
-        raise Errors::ParseInlineExtraCharactersAfterDelimiterError.new(@inline_scanner.line, @inline_scanner.pos,
-                                                                        @inline_scanner.remaining)
-      end
+      result = InlineParser.new(@line_scanner.read_next).parse
       unless result.is_a? Hash
         raise Errors::AssertionError,
               "Expected inline value to be Hash but is #{result.class.name}"
@@ -276,12 +240,7 @@ module NestedText
     end
 
     def parse_inline_list
-      @inline_scanner = InlineScanner.new(@line_scanner.read_next)
-      result = parse_inline
-      unless @inline_scanner.empty?
-        raise Errors::ParseInlineExtraCharactersAfterDelimiterError.new(@inline_scanner.line, @inline_scanner.pos,
-                                                                        @inline_scanner.remaining)
-      end
+      result = InlineParser.new(@line_scanner.read_next).parse
       unless result.is_a? Array
         raise Errors::AssertionError,
               "Expected inline value to be Array but is #{result.class.name}"

data/lib/nestedtext/scanners.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "nestedtext/errors_internal"
+require 'nestedtext/errors_internal'
 
 module NestedText
   class LineScanner
@@ -83,7 +83,7 @@ module NestedText
       :inline_dict, # {key1: value1, key2: value2}
       :inline_list, # [value1, value2]
       :unrecognized # could not be determined
-    ]
+    ].freeze
 
     attr_accessor :prev
     attr_reader :tag, :content, :indentation, :attribs, :lineno
@@ -112,12 +112,11 @@ module NestedText
     end
 
     def to_s
-      "[##{@lineno}] #{" " * @indentation}#{@content}"
+      "[##{@lineno}] #{' ' * @indentation}#{@content}"
     end
 
     private
 
-    # TODO: this regex must  unit tested.
     PATTERN_DICT_ITEM = /^
              (?<key>[^\s].*?)   # Key must start with a non-whitespace character, and goes until first
               \s*:              # first optional space, or :-separator
@@ -127,37 +126,44 @@ module NestedText
               )?
               $/x
 
-    def detect_line_tag_and_indentation
-      @indentation += 1 while @indentation < @content.length && @content[@indentation] == " "
+    def fast_forward_indentation
+      @indentation += 1 while @indentation < @content.length && @content[@indentation] == ' '
       @content = @content[@indentation..]
+    end
 
-      if @content.length == 0
+    def detect_line_tag
+      if @content.length.zero?
         self.tag = :blank
-      elsif @content[0] == "#"
+      elsif @content[0] == '#'
         self.tag = :comment
       elsif @content =~ /^:(?: |$)/
         self.tag = :key_item
-        @attribs["key"] = @content[2..] || ""
+        @attribs['key'] = @content[2..] || ''
       elsif @content =~ /^-(?: |$)/
         self.tag = :list_item
-        @attribs["value"] = @content[2..]
+        @attribs['value'] = @content[2..]
       elsif @content =~ /^>(?: |$)/
         self.tag = :string_item
-        @attribs["value"] = @content[2..] || ""
-      elsif @content[0] == "{"
+        @attribs['value'] = @content[2..] || ''
+      elsif @content[0] == '{'
         self.tag = :inline_dict
-      elsif @content[0] == "["
+      elsif @content[0] == '['
         self.tag = :inline_list
       elsif @content =~ PATTERN_DICT_ITEM
         self.tag = :dict_item
-        @attribs["key"] = Regexp.last_match(:key)
-        @attribs["value"] = Regexp.last_match(:value)
+        @attribs['key'] = Regexp.last_match(:key)
+        @attribs['value'] = Regexp.last_match(:value)
       else
         # Don't raise error here, as this line might not have been consumed yet,
         # thus could hide an error that we detect when parsing the previous line.
         self.tag = :unrecognized
       end
     end
+
+    def detect_line_tag_and_indentation
+      fast_forward_indentation
+      detect_line_tag
+    end
   end
   private_constant :Line
 end

data/lib/nestedtext/version.rb

@@ -2,5 +2,5 @@
 
 module NestedText
   # The version of this library.
-  VERSION = "3.2.1"
+  VERSION = '4.2.0'
 end

data/lib/nestedtext.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require_relative "nestedtext/core_ext"
-require_relative "nestedtext/decode"
-require_relative "nestedtext/encode"
-require_relative "nestedtext/encode_helpers"
-require_relative "nestedtext/error"
-require_relative "nestedtext/version"
+require_relative 'nestedtext/core_ext'
+require_relative 'nestedtext/decode'
+require_relative 'nestedtext/encode'
+require_relative 'nestedtext/encode_helpers'
+require_relative 'nestedtext/error'
+require_relative 'nestedtext/version'
 
 ##
 # # NestedText

data/nestedtext.gemspec

@@ -1,27 +1,28 @@
 # frozen_string_literal: true
 
-require_relative "lib/nestedtext/version"
+require_relative 'lib/nestedtext/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "nestedtext"
+  spec.name          = 'nestedtext'
   spec.version       = NestedText::VERSION
-  spec.authors       = ["Erik Westrup"]
-  spec.email         = ["erik.westrup@gmail.com"]
+  spec.authors       = ['Erik Westrup']
+  spec.email         = ['erik.westrup@gmail.com']
 
-  spec.summary       = "A ruby library for the human friendly data format NestedText (https://nestedtext.org/)"
-  spec.description   = "A ruby library for the human friendly data format NestedText (https://nestedtext.org/). There is support for decoding a NestedText file or string to Ruby data structures, as well as encoding Ruby objects to a NestedText file or string. Furthermore there is support for serialization and deserialization of custom classes. Support for v3.2.1 of the data format will all official tests passing."
-  spec.homepage      = "https://github.com/erikw/nestedtext-ruby/"
-  spec.license       = "MIT"
-  spec.required_ruby_version = [">= 3.0", "< 4"]
+  spec.summary       = 'A ruby library for the human friendly data format NestedText (https://nestedtext.org/)'
+  spec.description   = 'A ruby library for the human friendly data format NestedText (https://nestedtext.org/). There is support for decoding a NestedText file or string to Ruby data structures, as well as encoding Ruby objects to a NestedText file or string. Furthermore there is support for serialization and deserialization of custom classes. Support for v3.2.1 of the data format will all official tests passing.'
+  spec.homepage      = 'https://github.com/erikw/nestedtext-ruby/'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = ['>= 3.0', '< 4']
 
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/erikw/nestedtext-ruby/"
-  spec.metadata["changelog_uri"] = "https://github.com/erikw/nestedtext-ruby/blob/main/CHANGELOG.md"
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/erikw/nestedtext-ruby/'
+  spec.metadata['changelog_uri'] = 'https://github.com/erikw/nestedtext-ruby/blob/main/CHANGELOG.md'
 
   # For push to GitHub packages to work.
   # Reference: https://github.community/t/unable-to-push-rubygem-to-package-registry-the-expected-resource-was-not-found/14596/7
   spec.metadata = {
-    "github_repo" => "git@github.com:erikw/nestedtext-ruby.git"
+    'github_repo' => 'git@github.com:erikw/nestedtext-ruby.git',
+    'rubygems_mfa_required' => 'true'
   }
 
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
@@ -29,8 +30,9 @@ Gem::Specification.new do |spec|
       f.match(%r{\A(?:lib/|CHANGELOG.md|CONTRIBUTING.md|LICENSE.txt|README.md|SECURITY.md|nestedtext.gemspec)})
     end
   end
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency "warning", "~> 1.2"
-  spec.add_runtime_dependency "word_wrap", "~> 1.0"
+  spec.add_runtime_dependency 'unicode_utils', '~> 1.4'
+  spec.add_runtime_dependency 'warning', '~> 1.2'
+  spec.add_runtime_dependency 'word_wrap', '~> 1.0'
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: nestedtext
 version: !ruby/object:Gem::Version
-  version: 3.2.1
+  version: 4.2.0
 platform: ruby
 authors:
 - Erik Westrup
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-27 00:00:00.000000000 Z
+date: 2022-02-08 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: unicode_utils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: warning
   requirement: !ruby/object:Gem::Requirement
@@ -64,6 +78,7 @@ files:
 - lib/nestedtext/encode_helpers.rb
 - lib/nestedtext/error.rb
 - lib/nestedtext/errors_internal.rb
+- lib/nestedtext/inline_parser.rb
 - lib/nestedtext/parser.rb
 - lib/nestedtext/scanners.rb
 - lib/nestedtext/version.rb
@@ -73,6 +88,7 @@ licenses:
 - MIT
 metadata:
   github_repo: git@github.com:erikw/nestedtext-ruby.git
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths: