compactdata 0.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c7a0c33715643aef4cce1e216b64a24536a75ab4c6cd4f0c274fc12b75b46351
+  data.tar.gz: eda5c5dcf280d1ab706a02018932b8afef8ece3582d2bfaec8d2d5f5550c470e
+SHA512:
+  metadata.gz: cb7c8d70c740071124fc27faf9f2bf45707c6c6de71037deb1469914c3a7e3ecf078b9b00fe78a193cb68227bb687aa8cc81e322275c4e431d7d9c171fb9c67a
+  data.tar.gz: 69d14879fdcaca94c22f94f7707ae64f6fdbf89b5b0338c98fc18c02b6b7ded1f5c8daf2bdac4ec2b413e94440203609e834683dd48e497fe48cd636c2d3f25a

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.idea
+*.gem
+
+# rspec failure tracking
+.rspec_status
+
+Gemfile.lock

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+0.0.1
+===
+- Initial release

data/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in compactdata.gemspec
+group :development, :test do
+  gem 'debase'
+  gem 'ruby-debug-ide'
+  gem 'pry'
+end
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 NUM Technology Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,55 @@
+# CompactData Parser
+
+This Ruby gem is a CompactData parser and requires Ruby 2.6.0.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'compactdata'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or clone the repository and build and install it yourself as:
+
+    $ rake install
+    
+    or if that fails:
+    
+    $ sudo rake install
+
+## Usage
+
+In `irb`:
+
+```ruby
+2.6.0 :001 > require 'compactdata'
+ => true
+2.6.0 :002 > CompactData.parse 'a=b'
+ => {"a"=>"b"}
+2.6.0 :003 >
+```
+In a Ruby file:
+
+```ruby
+require 'compactdata'
+
+str='s=CompactData'
+pretty = true
+result = CompactData.parse str
+puts result
+
+```
+where `str` is the CompactData to be parsed. 
+
+## Development
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/compactdata.gemspec

@@ -0,0 +1,26 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'COMPACTDATA/VERSION'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'compactdata'
+  spec.version       = COMPACTDATA::VERSION
+  spec.authors       = ['Elliott Brown']
+  spec.email         = ['mail@elliottinvent.com']
+
+  spec.summary       = 'A Ruby parser for the CompactData serialisation format.'
+  spec.homepage      = 'https://gitlab.com/NUMTechnology/CompactData/Libraries/compactdata-ruby'
+  spec.license       = 'MIT'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'rake', '~> 12.3'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+end

data/lib/compactdata/interpreter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'json'
+
+# The CompactData namespace
+module CompactData
+  # Interpreter-specific errors
+  class ParserError < StandardError
+  end
+
+  # CompactData String to Ruby Hash/Array/primitive
+  def self.parse(str)
+    CompactData::Parser.parse_compactdata(str).to_j
+  end
+
+  # Ruby Hash/Array/primitive to CompactData
+  def self.generate(obj)
+    CompactData::Model::CompactData.new(CompactData::Model.to_compactdata(obj)).to_m
+  end
+end

data/lib/compactdata/model/model.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+module CompactData
+  # A Model module for CompactData
+  module Model
+    # The root of a CompactData object
+    class CompactData
+      attr_reader :value
+
+      def initialize(value)
+        @value = value
+      end
+
+      def to_s
+        if @value.instance_of? Array
+          @value.map(&:to_s)
+        else
+          "CompactData: #{@value}"
+        end
+      end
+
+      def to_j
+        @value.nil? ? 'null' : @value.to_j
+      end
+
+      def to_m
+        if @value.instance_of?(CompactDataMap)
+          @value.items.map(&:to_m).join(';')
+        else
+          @value.to_m
+        end
+      end
+    end
+
+    # A CompactData Array
+    class CompactDataArray
+      attr_reader :items
+
+      def initialize(items)
+        @items = items
+      end
+
+      def to_s
+        items = @items.map(&:to_s)
+        "CompactDataArray: #{items}"
+      end
+
+      def to_j
+        @items.map(&:to_j)
+      end
+
+      def to_m
+        list = @items.map(&:to_m).join ';'
+        "[#{list}]"
+      end
+    end
+
+    # A CompactData Map
+    class CompactDataMap
+      attr_reader :items
+
+      def initialize(items)
+        @items = items
+      end
+
+      def to_s
+        items = @items.map(&:to_s)
+        "CompactDataMap: #{items}"
+      end
+
+      def to_j
+        result = @items.map { |p| [p.key, p.value.to_j] }
+        result.to_h
+      end
+
+      def to_m
+        compactdata = @items.map(&:to_m).join ';'
+        "(#{compactdata})"
+      end
+    end
+
+    # A CompactData Primitive
+    class CompactDataPrimitive
+      attr_reader :value
+
+      def initialize(value)
+        @value = value
+      end
+
+      def to_s
+        "CompactDataPrimitive: #{@value}"
+      end
+
+      def to_m
+        @value.to_s
+      end
+    end
+
+    # A CompactData PAir
+    class CompactDataPair
+      attr_reader :key, :value
+
+      def initialize(key, value)
+        @key = key
+        @value = value
+      end
+
+      def to_s
+        "CompactDataPair: #{key}=#{@value}"
+      end
+
+      def to_j
+        { @key => @value.to_j }
+      end
+
+      def to_m
+        key = UTIL.non_string_primitive?(@key) ? "\"#{@key}\"" : @key
+        if @value.instance_of?(CompactDataPair)
+          "#{key}(#{@value.to_m})"
+        elsif @value.instance_of?(CompactDataMap) || @value.instance_of?(CompactDataArray)
+          "#{key}#{@value.to_m}"
+        else
+          "#{key}=#{@value.to_m}"
+        end
+      end
+    end
+
+    # A CompactData FLOAT
+    class CompactDataFloat
+      attr_reader :value
+
+      def initialize(value)
+        @value = value
+      end
+
+      def to_s
+        "CompactDataFloat: #{@value}"
+      end
+
+      def to_j
+        @value
+      end
+
+      def to_m
+        format('%.16G', @value).sub('E+', 'E')
+      end
+    end
+
+    # A CompactData Integer
+    class CompactDataInteger
+      attr_reader :value
+
+      def initialize(value)
+        @value = value
+      end
+
+      def to_s
+        "CompactDataInteger: #{@value}"
+      end
+
+      def to_j
+        @value
+      end
+
+      def to_m
+        @value.to_s
+      end
+    end
+
+    # A CompactData String
+    class CompactDataString
+      attr_reader :value
+
+      def initialize(value)
+        @value = value
+      end
+
+      def to_s
+        "CompactDataString: #{@value}"
+      end
+
+      def to_j
+        @value
+      end
+
+      def to_m
+        UTIL.escape_and_quote @value
+      end
+    end
+
+    # A CompactData Quoted String
+    class CompactDataQuoted
+      attr_reader :value
+
+      def initialize(value)
+        @value = value
+      end
+
+      def to_s
+        "CompactDataQuoted: #{@value}"
+      end
+
+      def to_j
+        @value
+      end
+
+      def to_m
+        UTIL.escape_and_quote @value
+      end
+    end
+
+    # A CompactData Boolean or NULL
+    class CompactDataBoolNull
+      def initialize(value)
+        @value = value
+      end
+
+      CompactData_NULL = CompactDataBoolNull.new nil
+      CompactData_TRUE = CompactDataBoolNull.new true
+      CompactData_FALSE = CompactDataBoolNull.new false
+
+      def to_s
+        "CompactDataBoolNull: #{@value}"
+      end
+
+      def to_j
+        @value
+      end
+
+      def to_m
+        @value.nil? ? 'null' : @value
+      end
+    end
+
+    # Convert a Ruby Hash/Array/Primitive to a CompactData object.
+    def self.to_compactdata(obj)
+      if obj.instance_of? Float
+        CompactDataFloat.new obj
+      elsif obj.instance_of? Integer
+        CompactDataInteger.new obj
+      elsif obj.instance_of? String
+        CompactDataString.new obj
+      elsif obj.instance_of? Hash
+        if obj.keys.length == 1
+          key = obj.keys[0]
+          CompactDataPair.new key, to_compactdata(obj[key])
+        else
+          CompactDataMap.new(obj.keys.map { |k| CompactDataPair.new k, to_compactdata(obj[k]) })
+        end
+      elsif obj.instance_of? Array
+        CompactDataArray.new(obj.map { |i| to_compactdata i })
+      elsif obj.instance_of? TrueClass
+        CompactDataBoolNull::CompactData_TRUE
+      elsif obj.instance_of? FalseClass
+        CompactDataBoolNull::CompactData_FALSE
+      elsif obj.instance_of? NilClass
+        CompactDataBoolNull::CompactData_NULL
+      else
+        puts "Cannot handle #{obj}"
+        exit
+      end
+    end
+  end
+end

data/lib/compactdata/parser/parser.rb

@@ -0,0 +1,294 @@
+# frozen_string_literal: true
+module CompactData
+  # A Parser module
+  module Parser
+    REPLACEMENTS = {
+      '\t' => "\t",
+      '\n' => "\n",
+      '\b' => "\b",
+      '\f' => "\f",
+      '\r' => "\r",
+      '~t' => "\t",
+      '~n' => "\n",
+      '~b' => "\b",
+      '~f' => "\f",
+      '~r' => "\r",
+      '~\\' => '\\',
+      '\\\\' => '\\',
+      '~~' => '~',
+      '\~' => '~',
+      '~(' => '(',
+      '\(' => '(',
+      '~)' => ')',
+      '\)' => ')',
+      '~}' => '}',
+      '\}' => '}',
+      '~[' => '[',
+      '\[' => '[',
+      '~]' => ']',
+      '\]' => ']',
+      '~;' => ';',
+      '\;' => ';',
+      '~`' => '`',
+      '\`' => '`',
+      '~"' => '"',
+      '"' => '"',
+      '~=' => '=',
+      '\=' => '='
+    }.freeze
+
+    def self.parse_compactdata(str)
+      tokens = CompactData::Tokeniser.tokenise str
+      if tokens.nil? || tokens.empty?
+        CompactData::Model::CompactData.new nil
+      elsif root_primitive? tokens[0]
+        CompactData::Model::CompactData.new tokens[0].value
+      else
+        result = parse_structures(tokens)
+        return CompactData::Model::CompactData.new result[0] if result.length == 1
+
+        return CompactData::Model::CompactData.new CompactData::Model::CompactDataMap.new(result) if all_pairs?(result)
+
+        CompactData::Model::CompactData.new CompactData::Model::CompactDataArray.new(result)
+      end
+    end
+
+    def self.root_primitive?(token)
+      %i[string quoted null true false integer float].include?(token)
+    end
+
+    def self.parse_structures(tokens)
+      result = []
+      until tokens.empty?
+        result.push parse_compactdata_value(tokens)
+        expect_separator = tokens.shift
+        if !expect_separator.nil? && expect_separator.type != :struct_sep
+          raise ParserError, "Expected ';' near #{tokens}"
+        end
+      end
+      result
+    end
+
+    def self.parse_compactdata_value(tokens)
+      first_token = tokens.shift
+
+      case first_token.type
+      when :lbracket
+        tokens.unshift first_token
+        return parse_compactdata_array tokens
+      when :lparen
+        tokens.unshift first_token
+        return parse_compactdata_map tokens
+      when :string, :quoted
+        peek = tokens[0]
+        key = first_token.value
+        if !peek.nil? && peek.type == :equals
+          tokens.shift
+          return CompactData::Model::CompactDataPair.new replace_escapes(unquote(key)), parse_pair_value(tokens)
+        end
+
+        if !peek.nil? && (peek.type == :lbracket || peek.type == :lparen)
+          return CompactData::Model::CompactDataPair.new replace_escapes(unquote(key)), parse_pair_value(tokens)
+        end
+
+        if peek.nil? || peek.type == :struct_sep || peek.type == :rparen || peek.type == :rbracket
+          return CompactData::Model::CompactDataString.new(replace_escapes(first_token.value)) if first_token.type == :string 
+
+          return CompactData::Model::CompactDataQuoted.new(replace_escapes(unquote(first_token.value)))
+        end
+      when :integer
+        return CompactData::Model::CompactDataInteger.new first_token.value
+      when :float
+        return CompactData::Model::CompactDataFloat.new first_token.value
+      when :null
+        return CompactData::Model::CompactDataBoolNull::CompactData_NULL
+      when true
+        return CompactData::Model::CompactDataBoolNull::CompactData_TRUE
+      when false
+        return CompactData::Model::CompactDataBoolNull::CompactData_FALSE
+      else
+        tokens.unshift first_token
+        maybe_primitive = parse_primitive tokens
+        return maybe_primitive unless maybe_primitive.nil?
+      end
+      raise ParserError, "Unexpected token: \'#{first_token}\'"
+    end
+
+    def self.parse_pair_value(tokens)
+      first_token = tokens.shift
+      case first_token.type
+      when :lbracket
+        tokens.unshift first_token
+        return parse_compactdata_array tokens
+      when :lparen
+        tokens.unshift first_token
+        return parse_compactdata_map tokens
+      when :string, :quoted
+        peek = tokens[0]
+
+        raise ParserError, "Unexpected token: \'#{first_token}\'" if !peek.nil? && peek.type == :equals
+
+        if !peek.nil? && (peek.type == :lbracket || peek.type == :lparen)
+          raise ParserError, "Unexpected token: \'#{first_token}\'"
+        end
+
+        if peek.nil? || peek.type == :struct_sep || peek.type == :rparen || peek.type == :rbracket
+          return CompactData::Model::CompactDataString.new replace_escapes(first_token.value) if first_token.type == :string
+
+          return CompactData::Model::CompactDataQuoted.new replace_escapes(unquote(first_token.value))
+        end
+        raise ParserError, "Unexpected token: \'#{first_token}\'"
+      when :integer
+        return CompactData::Model::CompactDataInteger.new first_token.value
+      when :float
+        return CompactData::Model::CompactDataFloat.new first_token.value
+      when :null
+        return CompactData::Model::CompactDataBoolNull::CompactData_NULL
+      when true
+        return CompactData::Model::CompactDataBoolNull::CompactData_TRUE
+      when false
+        return CompactData::Model::CompactDataBoolNull::CompactData_FALSE
+      else
+        tokens.unshift first_token
+        maybe_primitive = parse_primitive tokens
+        return maybe_primitive unless maybe_primitive.nil?
+      end
+      raise ParserError, "Unexpected token: \'#{first_token}\'"
+    end
+
+    def self.parse_primitive(tokens)
+      result = nil
+      tok = tokens.shift
+
+      case tok.type
+      when :lparen, :rparen, :lbracket, :rbracket, :equals
+        raise ParserError, "Unexpected token: \'#{tok}\'" if tokens.empty?
+
+        tokens.unshift tok
+        return nil
+      when :null
+        result = CompactData::Model::CompactDataBoolNull::CompactData_NULL
+      when true
+        result = CompactData::Model::CompactDataBoolNull::CompactData_TRUE
+      when false
+        result = CompactData::Model::CompactDataBoolNull::CompactData_FALSE
+      when :quoted
+        result = CompactData::Model::CompactDataQuoted.new replace_escapes(unquote(tok.value))
+      when :string
+        result = CompactData::Model::CompactDataString.new replace_escapes(tok.value)
+      when :integer
+        result = CompactData::Model::CompactDataInteger.new tok.value
+      when :float
+        result = CompactData::Model::CompactDataFloat.new tok.value
+      else
+        raise ParserError, "Unknown token type in: \'#{tok}\'"
+      end
+
+      peek = tokens[0]
+      raise ParserError, 'Only one primitive allowed at the root.' if !peek.nil? && peek.type == :struct_sep
+
+      if !peek.nil? && (peek.type == :lparen || peek.type == :lbracket || peek.type == :equals)
+        s.unshift tok
+        return nil
+      elsif !peek.nil?
+        raise ParserError, "Unexpected token: \'#{peek}\'"
+      end
+      result
+    end
+
+    def self.parse_compactdata_map(tokens)
+      first_token = tokens.shift
+      entries = []
+
+      until tokens.empty?
+        peek = tokens[0]
+        if !peek.nil? && peek.type == :rparen
+          tokens.shift
+          break
+        end
+
+        mp = parse_compactdata_value tokens
+        entries.push mp
+
+        peek = tokens[0]
+
+        raise ParserError, "Expected ')' near #{first_token}" if peek.nil?
+
+        case peek.type
+        when :rparen
+          tokens.shift
+          break
+        when :struct_sep
+          tokens.shift
+          peek = tokens[0]
+          raise ParserError, "Unexpected ; before ] at #{peek}" if !peek.nil? && peek.type == :rparen
+        end
+      end
+      CompactData::Model::CompactDataMap.new entries
+    end
+
+    def self.parse_compactdata_array(tokens)
+      first_token = tokens.shift
+      entries = []
+
+      until tokens.empty?
+        peek = tokens[0]
+        if !peek.nil? && peek.type == :rbracket
+          tokens.shift
+          break
+        end
+
+        mp = parse_compactdata_value tokens
+        entries.push mp
+
+        peek = tokens[0]
+
+        raise ParserError, "Expected ']' near #{first_token}" if peek.nil?
+
+        case peek.type
+        when :rbracket
+          tokens.shift
+          break
+        when :struct_sep
+          tokens.shift
+          peek = tokens[0]
+          raise ParserError, "Unexpected ; before ] at #{peek}" if !peek.nil? && peek.type == :rparen
+        end
+      end
+      CompactData::Model::CompactDataArray.new entries
+    end
+
+    def self.unquote(str)
+      return str unless str.instance_of?(String)
+
+      if (str.start_with?('`') && str.end_with?('`')) || (str.start_with?('"') && str.end_with?('"'))
+        str[1..-2]
+      else
+        str
+      end
+    end
+
+    def self.replace_escapes(str)
+      return str unless str.instance_of?(String)
+
+      result = str
+      i = 0
+      while i < str.length
+        REPLACEMENTS.each_pair do |key, value|
+          if result.slice(i..).start_with?(key)
+            result = result.sub(key, value)
+            break
+          end
+        end
+        i += 1
+      end
+      result
+    end
+
+    def self.all_pairs?(arr)
+      result = true
+      arr.each { |i| result = false unless i.instance_of? CompactData::Model::CompactDataPair }
+      result
+    end
+  end
+end

data/lib/compactdata/tokeniser/context.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module CompactData
+  module Tokeniser
+    # A parsing context for a CompactData String
+    class Context
+      WS = " \t\r\n"
+      INTEGER_REGEX = '^-?(?:0|[1-9]\d*)$'
+      FLOAT_REGEX = '^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$'
+      NON_STRING_TOKENS = '[]();"=`'
+
+      def initialize(str)
+        @str = str
+        @tokens = []
+        @tok_start = 0
+      end
+
+      def parse
+        while next_token
+        end
+        @tokens
+      end
+
+      def next_token
+        while @tok_start < @str.length
+          break unless WS.include?(@str[@tok_start])
+
+          @tok_start += 1
+        end
+
+        return false if @tok_start >= @str.length
+
+        case @str[@tok_start]
+        when '('
+          tok_type = :lparen
+          tok_end = @tok_start + 1
+        when ')'
+          tok_type = :rparen
+          tok_end = @tok_start + 1
+        when '['
+          tok_type = :lbracket
+          tok_end = @tok_start + 1
+        when ']'
+          tok_type = :rbracket
+          tok_end = @tok_start + 1
+        when ';'
+          tok_type = :struct_sep
+          tok_end = @tok_start + 1
+        when '='
+          tok_type = :equals
+          tok_end = @tok_start + 1
+        when '"'
+          tok_type = :quoted
+          tok_end = scan_to_end_of_quoted(@str, @tok_start, '"')
+        when '`'
+          tok_type = :quoted
+          tok_end = scan_to_end_of_quoted(@str, @tok_start, '`')
+        when '{'
+          ## Braced strings must include the braces in the string
+          tok_type = :quoted
+          tok_end = scan_to_end_of_quoted(@str, @tok_start, '}')
+        else
+          tok_type = :string
+          tok_end = scan_to_end_of_string(@str, @tok_start)
+        end
+
+        tok_value = @str[@tok_start..(tok_end - 1)].strip
+
+        if tok_value.match? INTEGER_REGEX
+          number = tok_value.to_i
+          @tokens.push Token.new(:integer, number, @tok_start, tok_end)
+        elsif tok_value.match? FLOAT_REGEX
+          number = tok_value.to_f
+          @tokens.push Token.new(:float, number, @tok_start, tok_end)
+        elsif tok_value == 'null'
+          @tokens.push Token.new(:null, nil, @tok_start, tok_end)
+        elsif tok_value == 'true'
+          @tokens.push Token.new(true, true, @tok_start, tok_end)
+        elsif tok_value == 'false'
+          @tokens.push Token.new(false, false, @tok_start, tok_end)
+        else
+          @tokens.push Token.new(tok_type, tok_value, @tok_start, tok_end)
+        end
+
+        @tok_start = tok_end
+        tok_end < @str.length
+      end
+
+      def scan_to_end_of_quoted(str, start, quote_char)
+        end_str = start + 1
+        while end_str < str.length
+          end_char = str[end_str]
+          prev_char = str[end_str - 1]
+
+          break if end_char == quote_char && prev_char != '\\' && prev_char != '~'
+
+          end_str += 1
+        end
+        end_str + 1
+      end
+
+      def scan_to_end_of_string(str, start)
+        end_str = start + 1
+        while end_str < str.length
+          break if NON_STRING_TOKENS.include?(str[end_str]) && !escaped?(str, end_str - 1)
+
+          end_str += 1
+        end
+        end_str
+      end
+
+      def escaped?(str, pos)
+        pos >= 0 && (str[pos] == '~' || str[pos] == '\\')
+      end
+    end
+  end
+end

data/lib/compactdata/tokeniser/tokeniser.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'compactdata/tokeniser/context'
+
+module CompactData
+  # A CompactData Tokeniser module
+  module Tokeniser
+    # A class to represent CompactData tokens
+    class Token
+      attr_reader :type, :value, :from, :to
+
+      def initialize(type, value, from, to)
+        @type = type
+        @value = value
+        @from = from
+        @to = to
+      end
+
+      def to_s
+        "type: #{@type}, from: #{@from}, to: #{@to}, value: \"#{@value}\""
+      end
+    end
+
+    def self.tokenise(str)
+      Context.new(str).parse
+    end
+  end
+end

data/lib/compactdata/util/functions.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module CompactData
+  # Helper functions for CompactData.generate
+  module UTIL
+    SHOULD_BE_GRAVE_QUOTED = /.*[()\[\];{}="].*/.freeze
+    IS_NUMERIC = /^-?[0-9]*\.?[0-9]+(?:[Ee][+-]?[0-9]+)?$/.freeze
+
+    def self.escape_graves(str)
+      return str unless str
+
+      str.gsub(/`/, '~`')
+    end
+
+    def self.escape_double_quotes(str)
+      return str unless str
+
+      i = str.index('"', 1) # ignore the initial quote
+      result = str
+      # Don't affect the final quote in the string
+      while !i.nil? && i < (result.length - 1)
+        result = result.slice(0..(i - 1)) + '~u0022' + result.slice((i + 1)..)
+        i = result.index('"', 1)
+      end
+      result
+    end
+
+    def self.grave_quote_if_necessary(str)
+      return str unless str
+
+      if str.match?(SHOULD_BE_GRAVE_QUOTED) ||
+         (str.match?(IS_NUMERIC) && str != '00' && str != '01' && str != '000') ||
+         (str.strip.empty? ||
+          str.match?(IS_NUMERIC) ||
+          str == 'true' ||
+          str == 'false' ||
+          str == 'null')
+        "`#{str}`"
+      else
+        str
+      end
+    end
+
+    def self.double_quote_if_necessary(str)
+      if str && str.include?('~`')
+        "\"#{str}\""
+      else
+        str
+      end
+    end
+
+    def self.replace_nbsp(str)
+      str.gsub('\u00a0', ' ')
+    end
+
+    def self.escape_and_quote(str)
+      double_quote_if_necessary(
+        grave_quote_if_necessary(
+          escape_graves(
+            UNICODE.escape(
+              replace_nbsp(
+                escape_double_quotes(str)
+              )
+            )
+          )
+        )
+      )
+    end
+
+    def self.non_string_primitive?(str)
+      str == 'true' || str == 'false' || str == 'null' || str.match?(IS_NUMERIC)
+    end
+  end
+end

data/lib/compactdata/util/unicode.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: false
+
+module CompactData
+  # Handle escaping of unicode characters.
+  module UNICODE
+    VERTICAL_TAB = 0x0b
+    BACKSPACE = 0x08
+    FORMFEED = 0x0c
+    LINEFEED = 0x0a
+    CARRIAGE_RETURN = 0x0d
+    TAB = 0x09
+
+    def self.escape_char(chr)
+      return '' unless chr
+
+      if chr >= 32 && chr <= 127
+        '' << chr
+      elsif chr == VERTICAL_TAB
+        '~u000B'
+      elsif chr == BACKSPACE
+        '\\b'
+      elsif chr == FORMFEED
+        '\\f'
+      elsif chr == LINEFEED
+        '\\n'
+      elsif chr == CARRIAGE_RETURN
+        '\\r'
+      elsif chr == TAB
+        '\\t'
+      else
+        '' << chr
+      end
+    end
+
+    def self.escape(str)
+      result = ''
+
+      str.each_codepoint do |c|
+        result << escape_char(c)
+      end
+      result
+    end
+  end
+end

data/lib/compactdata/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module COMPACTDATA
+  VERSION = '0.0.6'
+end

data/lib/compactdata.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'compactdata/version'
+require 'compactdata/model/model'
+require 'compactdata/tokeniser/tokeniser'
+require 'compactdata/parser/parser'
+require 'compactdata/util/unicode'
+require 'compactdata/util/functions'
+require 'compactdata/interpreter'

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: compactdata
+version: !ruby/object:Gem::Version
+  version: 0.0.6
+platform: ruby
+authors:
+- Elliott Brown
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-03-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description:
+email:
+- mail@elliottinvent.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".DS_Store"
+- ".gitignore"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- compactdata.gemspec
+- lib/compactdata.rb
+- lib/compactdata/interpreter.rb
+- lib/compactdata/model/model.rb
+- lib/compactdata/parser/parser.rb
+- lib/compactdata/tokeniser/context.rb
+- lib/compactdata/tokeniser/tokeniser.rb
+- lib/compactdata/util/functions.rb
+- lib/compactdata/util/unicode.rb
+- lib/compactdata/version.rb
+homepage: https://gitlab.com/NUMTechnology/CompactData/Libraries/compactdata-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: A Ruby parser for the CompactData serialisation format.
+test_files: []