simple_lexer 0.0.2 → 0.0.3

data/README.md

@@ -1,6 +1,6 @@
 # SimpleLexer
 
-TODO: Write a gem description
+A very basic toy Lexer implemented with Regular Expressions.
 
 ## Installation
 
@@ -18,7 +18,7 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+See docs for SimpleLexer::Lexer.
 
 ## Contributing
 

data/lib/simple_lexer.rb

@@ -2,18 +2,31 @@ require_relative "simple_lexer/version"
 
 module SimpleLexer 
 
+
+  # An Exception that is raised when Lexer encounters text for which 
+  # there is no rule to match. 
   class NoMatchError < Exception
-    # unable to match
   end
 
+  # Exception that is raised when Lexer is finished tokenizing the
+  # input string.
   class EndOfStreamException < Exception
-    # when the Lexer is finished
   end
 
+  # Object defined with certain rules that takes text as input and
+  # outputs Tokens based on the rules.
+  # @!attribute [r] rules
+  #   @return [Array<Regexp>] A list of the rules for the Lexer.
+  # @!attribute [rw] pos 
+  #   @return [Fixnum] The current position of the input pointer. 
   class Lexer
 
-    attr_reader :rules, :pos
+    attr_reader :rules
+    attr_accessor :pos
 
+    # Creates a new instance of Lexer.
+    # @yield [] Some rules passed to instance_eval.
+    # @see #tok An example of a number Lexer using <code>tok</code>. 
     def initialize(&rules)
       @rules = [] # list of {:rule => Regexp, :token => :token_id}
       @ignore = [] # list of Regexp
@@ -21,24 +34,37 @@ module SimpleLexer
       instance_eval &rules
     end
 
+    # Defines a new Token rule for the Lexer to match.
+    # @param [Regexp] rule Regular expression that defines the token 
+    # @param [Symbol] token Token class 
+    # @yield [text] The expression will give the Token its value. 
+    # @example Rule for numbers
+    #   my_lexer = SimpleLexer::Lexer.new do
+    #     tok /-?\d+(\.\d+)?/, :number do |text| text.to_f end
+    #   end
+    #   my_lexer.load = "-435.234"
+    #   puts my_lexer.next_token[:value] # -435.234
     def tok(rule, token, &action)
-      # defining a new rule:
-      #
-      # my_lexer = SimpleLexer::Lexer.new do
-      #   tok /\w+/, :identifier
-      # end
-       
       @rules << {:rule => Regexp.new('\A' + rule.source), :token => token, :action => action}
     end
 
-    def ign(rule)
-      # defining conditions to ignore:
-      #
-      # my_lexer = SimpleLexer::Lexer.new do
-      #   tok /\w+/, :identifier
-      #   ign :whitespace
-      # end
-      
+    # Defines rules of input classes to ignore (consume and not output any
+    # tokens.)
+    # @param [Regexp, Symbol] rule Regular expression that defines ignored
+    #   characters.
+    # @note You can set _rule_ to <code>:whitespace</code> to ignore whitespace
+    #     characters.
+    # @example Ignoring parentheses
+    #   my_lexer = SimpleLexer::Lexer.new do
+    #     tok /\w+/, :identifier
+    #     ign /[\(\)]/
+    #   end
+    # @example Ignoring whitespace
+    #   my_lexer = SimpleLexer::Lexer.new do
+    #     tok /\w+/, :identifier
+    #     ign :whitespace
+    #   end
+    def ign(rule) 
       if rule == :whitespace
         rule = /\s+/
       end
@@ -46,21 +72,26 @@ module SimpleLexer
       @ignore << Regexp.new('\A' + rule.source)
     end
 
-    def load=(string)
-      # load a string into the lexer
-      # my_lexer.load( ... )
-      
-      @load = string
+    # Give the Lexer some text to tokenize.
+    # @param [String] input Text for the Lexer to tokenize.
+    def load=(input)
+      @load = input 
       @pos = 0 
     end
-    
+   
+    # What still remains to be processed.
+    # @return [String] Substring of the input starting from input pointer.
     def load 
-      # what the lexer currently sees
-      # my_lexer.load ...
-      
       @load[@pos..-1]
     end
 
+    # Gets the next Token in the input and advances the input pointer.
+    # @return [Hash{Symbol=>Values}]
+    #   - <code>:token</code> Token class
+    #   - <code>:text</code> Matched text
+    #   - <code>:value</code> Value as defined by passed block, if applicable.
+    # @raise [NoMatchError] If load contains a sequence for which the Lexer has
+    #   no rule.
     def next_token
       # get the next token
       # my_lexer.next_token -> [ :token => :token_id, :text => matched ]
@@ -85,10 +116,9 @@ module SimpleLexer
       raise NoMatchError, "Unable to match, unexpected characters: '#{load[0..10]}...'"
     end
 
-    def all_tokens 
-      # returns the array of all tokens until it is finished lexing
-      # my_lexer.all_tokens
-      
+    # Tokenize the entire input stream.
+    # @return [Array<Hash>] An Array of Tokens processed by the Lexer
+    def all_tokens  
       tokens = []  
       loop do
         tokens << next_token
@@ -97,24 +127,11 @@ module SimpleLexer
       tokens 
     end
 
+    # Checks if the Lexer has finished Tokenizing the entire input stream.
+    # @return [Boolean] Whether Lexer has reached the end of input.
     def finished?
       return @pos >= @load.length
     end
 
   end
 end
-
-my_lexer = SimpleLexer::Lexer.new do
-  tok /-?\d+(\.\d+)?/, :number do |t| t.to_f end
-  tok /\+/, :plus
-  tok /-/, :minus
-  tok /\//, :div
-  tok /\*/, :mult
-  tok /\(/, :lparen
-  tok /\)/, :rparen
-  
-  ign :whitespace
-end
-
-my_lexer.load = "321.32 + -432.388 - 33/4.3 - 4.228 * 5 - (32*632)"
-p my_lexer.all_tokens

data/lib/simple_lexer/version.rb

@@ -1,3 +1,3 @@
 module SimpleLexer
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simple_lexer
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &11890980 !ruby/object:Gem::Requirement
+  requirement: &10265840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '1.3'
   type: :development
   prerelease: false
-  version_requirements: *11890980
+  version_requirements: *10265840
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &11890240 !ruby/object:Gem::Requirement
+  requirement: &10264520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *11890240
+  version_requirements: *10264520
 description: A simple toy lexer for Ruby
 email:
 - wchen298@gmail.com
@@ -74,3 +74,4 @@ signing_key:
 specification_version: 3
 summary: Rudimentary lexer for Ruby
 test_files: []
+has_rdoc: