rbscmlex 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d945c95432fbcd698af635fd0d4a9e0f54a468270fc6f6e8c5a69df7202b1f3
-  data.tar.gz: b359cb4c75e7082e961714d0994ed3d503a81796bbbf51c489c6e32c62dfc819
+  metadata.gz: ec8d6513252c0196840bfc388ab2cb429d5d1cb579e5fb23d034e6509ab24bf5
+  data.tar.gz: fd952eecb26646994dcfe9786fa5678b68f5ab04191009079f47acedaba1af3a
 SHA512:
-  metadata.gz: 6f3da67c6a2479b94fb9a7fff6c6c89c03e0f386f94418a16d3ff751e505839f62deb2437127607e7259d51e1c8204c7583e3d5207c944ba0114b996f7f212bb
-  data.tar.gz: c0a68a273da923404f3cc3051f5850250076edff9bb4637523f3b5667862d7c6b1d64b348998c0c8c2f8604f10daf364c3918928cccd0304f7ed6468585ca289
+  metadata.gz: 6344300170f133448eb6f6b1646ffa0e617a900c3df87b00f9a48cd6c008da73e8b9161cac4afb4d752a37c3981cd7ad1fa324c48044d484c26cd836bcfe5cc9
+  data.tar.gz: ecf7e267663faa64023c1f72ed82fac38b799d2fd370d7a6725e512780618ebbd70f121bf8031abb3738f9827dbd5e7c855ad700055b18dbbec2b79356d3f16b

data/CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 ## [Unreleased]
 - (nothing to record here)
 
+## [0.1.2] - 2021-05-07
+### Added
+- Add a mechanism to initialize a Parser instance from an array of
+  tokens, which already created from source of Scheme.
+
+### Fixed
+- Fix issue #3: a wrong link in `README.md`.
+
 ## [0.1.1] - 2021-05-06
 ### Fixed
 - Fix issue #1: `rbscmlex` fails to read from STDIN.

data/README.md

@@ -28,7 +28,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at [https://github.com/mnbi/rus3](https://github.com/mnbi/rus3).
+Bug reports and pull requests are welcome on GitHub at [https://github.com/mnbi/rbscmlex](https://github.com/mnbi/rbscmlex).
 
 
 ## License

data/lib/rbscmlex/lexer.rb

@@ -64,56 +64,136 @@ module Rbscmlex
 
     include Enumerable
 
-    def initialize(src, form: TOKEN_DEFAULT_FORM)
-      @form = form
-      @tokens = tokenize(src)
-      @size = @tokens.size
+    def initialize(obj, form: TOKEN_DEFAULT_FORM)
+      set_form(form)
+      init_pos
+      case obj
+      when String
+        # obj must be source program of Scheme.
+        @tokens = tokenize(obj)
+      when Array
+        # obj might be an array of tokens.
+        input_form = detect_form(obj[0])
+        case input_form
+        when :hash, :json, :token
+          @tokens = read_tokens(obj, form: input_form)
+        else
+          raise InvalidConversionTypeError, "cannot convert #{obj[0]} as token"
+        end
+      else
+        raise InvalidConversionTypeError, "cannot convert #{obj} as tokens"
+      end
+    end
 
-      @current_pos = @next_pos = 0
+    def [](index)
+      convert(@tokens[index])
     end
 
     def each(&blk)
       if block_given?
-        @tokens.each(&blk)
+        @tokens.each { |tk|
+          yield convert(tk)
+        }
         self
       else
-        @tokens.each
+        to_a.to_enum
       end
     end
 
     def to_a
-      @tokens
+      convert_all(@tokens)
     end
 
     def size
-      @size
+      @tokens.size
     end
 
     def current_token
-      @tokens[@current_pos]
+      self[@current_pos]
     end
 
     def next_token
       check_pos
       @current_pos = @next_pos
       @next_pos += 1
-      @tokens[@current_pos]
+      self[@current_pos]
     end
 
     def peek_token(num = 0)
       check_pos
-      @tokens[@next_pos + num]
+      self[@next_pos + num]
     end
 
     def rewind
-      @current_pos = @next_pos = 0
+      init_pos
       self
     end
 
+    # :stopdoc:
+
     private
 
+    def set_form(form)
+      if TOKEN_FORMS.include?(form)
+        @form = form
+      else
+        raise InvalidConversionTypeError, "cannot generate #{form} as token"
+      end
+    end
+
+    def init_pos
+      @current_pos = @next_pos = 0
+    end
+
+    def read_tokens(ary, form: :token)
+      conv_proc ={hash: :hash2token, json: :json2token, token: nil}[form]
+      conv_proc ? ary.map{|e| Rbscmlex.send(conv_proc, e)} : ary.dup
+    end
+
+    def detect_form(obj)
+      case obj
+      when Hash
+        valid_token?(obj) ? :hash : nil
+      when Token
+        :token
+      when String
+        begin
+          JSON.parse(obj, symbolize_names: true)
+        rescue JSON::ParserError => _
+          nil
+        else
+          :json
+        end
+      else
+        nil
+      end
+    end
+
+    def valid_token?(obj)
+      case obj
+      when Hash
+        obj.key?(:type) and obj.key?(:literal)
+      when Token
+        Rbscmlex.token_type?(obj.type)
+      else
+        false
+      end
+    end
+
+    def converter
+      { hash: :to_h, json: :to_json, token: nil}[@form]
+    end
+
+    def convert(token)
+      converter ? token.send(converter) : token
+    end
+
+    def convert_all(tokens)
+      converter ? token.map(&converter) : tokens
+    end
+
     def check_pos
-      raise StopIteration if @next_pos >= @size
+      raise StopIteration if @next_pos >= size
     end
 
     S2R_MAP = { "(" => "( ", ")" => " ) ", "'" => " ' " } # :nodoc:
@@ -124,32 +204,34 @@ module Rbscmlex
       cooked.split(" ").map { |literal|
         case literal
         when "("
-          Rbscmlex.new_token(:lparen, literal, @form)
+          Rbscmlex.new_token(:lparen, literal)
         when ")"
-          Rbscmlex.new_token(:rparen, literal, @form)
+          Rbscmlex.new_token(:rparen, literal)
         when "."
-          Rbscmlex.new_token(:dot, literal, @form)
+          Rbscmlex.new_token(:dot, literal)
         when "'"
-          Rbscmlex.new_token(:quotation, literal, @form)
+          Rbscmlex.new_token(:quotation, literal)
         when "#("
-          Rbscmlex.new_token(:vec_lparen, literal, @form)
+          Rbscmlex.new_token(:vec_lparen, literal)
         when BOOLEAN
-          Rbscmlex.new_token(:boolean, literal, @form)
+          Rbscmlex.new_token(:boolean, literal)
         when IDENTIFIER
-          Rbscmlex.new_token(:identifier, literal, @form)
+          Rbscmlex.new_token(:identifier, literal)
         when CHAR
-          Rbscmlex.new_token(:character, literal, @form)
+          Rbscmlex.new_token(:character, literal)
         when STRING
-          Rbscmlex.new_token(:string, literal, @form)
+          Rbscmlex.new_token(:string, literal)
         when ARITHMETIC_OPS, COMPARISON_OPS
-          Rbscmlex.new_token(:op_proc, literal, @form)
+          Rbscmlex.new_token(:op_proc, literal)
         when REAL_NUM, RATIONAL, COMPLEX, PURE_IMAG
-          Rbscmlex.new_token(:number, literal, @form)
+          Rbscmlex.new_token(:number, literal)
         else
-          Rbscmlex.new_token(:illegal, literal, @form)
+          Rbscmlex.new_token(:illegal, literal)
         end
       }
     end
 
+    # :startdoc:
+
   end
 end

data/lib/rbscmlex/token.rb

@@ -36,26 +36,26 @@ module Rbscmlex
   # a structure to store properties of a token of Scheme program.
 
   Token = Struct.new(:type, :literal) {
+    # :stopdoc:
+    # `to_a` and `to_h` are automatically defined for a class
+    # generated from Struct.
+    # :startdoc:
+
     alias :to_s :literal
+
+    # Generates a new string of JSON notation, which has "type" and
+    # "literal" as its key.
+    def to_json
+      JSON.generate(to_h)
+    end
   }
 
   class << self
 
-    # Instantiates a new token object form type and literal.  The 3rd
-    # argument specifies the form of the object.  It must be one of
-    # :token, :hash, and :json.
-
-    def new_token(type, literal = nil, form = :token)
-      case form
-      when :token
-        Token.new(type, literal)
-      when :hash
-        make_hash(type, literal)
-      when :json
-        JSON.generate(make_hash(type, literal))
-      else
-        raise InvalidConversionTypeError, "cannot generate #{type} as token"
-      end
+    # Instantiates a new token object form type and literal.
+
+    def new_token(type, literal = nil)
+      Token.new(type, literal)
     end
 
     # Returns true when the argument is valid token type.
@@ -64,7 +64,7 @@ module Rbscmlex
       TOKEN_TYPES.include?(type)
     end
 
-    # Returns a new Hash object with type and literal.
+    # Returns a new Hash object with type and literal as its keys.
 
     def make_hash(type, literal)
       {type: type, literal: literal}
@@ -75,28 +75,54 @@ module Rbscmlex
     # must be valid token type.  Otherwise, raises UnknownTokenTypeError.
 
     def hash2token(hash)
-      if h.key?("type") and h.key?("literal")
-        type = h["type"].intern
+      if hash.key?(:type) and hash.key?(:literal)
+        type = hash[:type].intern
         raise UnknownTokenTypeError, ("got=%s" % type) unless token_type?(type)
-        literal = h["literal"]
-        Token.new(type.intern, literal)
+        literal = hash[:literal]
+        new_token(type, literal)
       else
         raise InvalidHashError, ("got=%s" % hash)
       end
     end
 
     # Converts a JSON notation, which hash type and literal, to a new
-    # token object.  The value associated to type of the Hash must be
+    # token object.  The value associated to type of the JSON must be
     # valid token type.  Otherwise, raises UnknownTokenTypeError.
 
     def json2token(json)
-      h = JSON.parse(json)
+      h = JSON.parse(json, symbolize_names: true)
       begin
         hash2token(h)
       rescue InvalidHashError => _
         raise InvalidJsonError, ("got=%s" % json)
       end
     end
+
+    # Converts a Token object to a Hash object.
+
+    def token2hash(token)
+      token.to_h
+    end
+
+    # Converts a Token object to a string of JSON notation.
+
+    def token2json(token)
+      token.to_json
+    end
+
+    # Converts a Hash object to a string of JSON notation.
+
+    def hash2json(hash)
+      JSON.generate(hash)
+    end
+
+
+    # Converts a JSON notation to a new Hash object.
+
+    def json2hash(json)
+      JSON.parse(json)
+    end
+
   end
 
 end

data/lib/rbscmlex/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module Rbscmlex
-  VERSION = "0.1.1"
-  RELEASE = "2021-05-06"
+  VERSION = "0.1.2"
+  RELEASE = "2021-05-07"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbscmlex
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - mnbi
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-05-06 00:00:00.000000000 Z
+date: 2021-05-07 00:00:00.000000000 Z
 dependencies: []
 description: A simple lexical analyzer for Scheme
 email: