rlex 0.5.3 → 0.6.0

data/.gitignore

@@ -1,5 +1,6 @@
 *.gem
 *.rbc
+*~
 .bundle
 .config
 .yardoc

data/.yardopts

@@ -1 +1 @@
---no-private lib - LICENSE
+--no-private lib - LICENSE CHANGELOG.md

data/CHANGELOG.md

@@ -0,0 +1,22 @@
+# Changes
+
+## Version 0.6.0
+
+Produced tokens now include line number and column position of the matched text
+
+## Version 0.5.4
+
+Update gem spec to reflect development dependency on `rspec`
+
+## Version 0.5.3
+
+Add `require` statements internally so `require 'rlex'` is sufficient to use
+the gem
+
+## Version 0.5.2
+
+YARD docs generated correctly
+
+## Version 0.5.0
+
+Initial version

data/lib/rlex/lexer.rb

@@ -100,10 +100,11 @@ module Rlex
     #
     def keyword(name = nil, kword)
       # @todo Validate the keyword name
-      name = kword if name == nil
-      pattern = Regexp.new(Regexp.escape kword.to_s)
+      kword_str = kword.to_s
+      name = kword.to_sym if name == nil
+      pattern = Regexp.new(Regexp.escape kword_str)
       rule name, pattern
-      @keywords[kword.to_s] = Token.new name.to_sym, kword.to_s
+      @keywords[kword_str] = Token.new name.to_sym, kword_str
       return name.to_sym
     end
 
@@ -117,6 +118,8 @@ module Rlex
     # @return [String] The specified input
     #
     def start(input)
+      @line = 1
+      @col = 0
       @scanner = StringScanner.new input
       return input
     end
@@ -134,9 +137,11 @@ module Rlex
       return next_token if ignore_prefix?
       rule = greediest_rule
       if rule
-        prefix = @scanner.scan(rule.pattern)
+        prefix = fetch_prefix_and_update_pos(rule.pattern)
         keyword = @keywords[prefix]
-        return keyword ? keyword : Token.new(rule.name, prefix)
+        type = keyword ? keyword.type : rule.name
+        token = keyword ? keyword.value : prefix
+        return Token.new(type, token, @line, @col - token.size)
       end
       raise "unexpected input <#{@scanner.peek(5)}>"
     end
@@ -149,7 +154,7 @@ module Rlex
     # @private
     def ignore_prefix?
       @ignored.each do |pattern|
-        prefix = @scanner.scan(pattern)
+        prefix = fetch_prefix_and_update_pos(pattern)
         return true if prefix
       end
       return false
@@ -168,5 +173,21 @@ module Rlex
       end
       return r
     end
+
+    # @private
+    def fetch_prefix_and_update_pos(pattern)
+      prefix = @scanner.scan(pattern)
+      return nil if not prefix
+      parts = prefix.split("\n", -1) # arg -1 allows empty lines
+      if parts.count == 1
+        # Staying on the same line
+        @col += prefix.length
+      else
+        # On a new line
+        @line += parts.count - 1
+        @col = parts.last.length
+      end
+      return prefix
+    end
   end
 end

data/lib/rlex/token.rb

@@ -6,11 +6,13 @@ module Rlex
   # @attr_reader [Symbol] type Type of the token, such as the name of
   #   the rule used to match it
   # @attr_reader [String] value Text matched from the input
+  # @attr_reader [Integer] line Line number of the matched text
+  # @attr_reader [Integer] col Column position of the matched text
   #
-  Token = Struct.new :type, :value
+  Token = Struct.new :type, :value, :line, :col
 
   # Special token used when the lexer has reached the end of the
   # specified input.
   #
-  EOF_TOKEN = Token.new :eof, ""
+  EOF_TOKEN = Token.new :eof, "", -1, -1
 end

data/lib/rlex/version.rb

@@ -1,4 +1,4 @@
 module Rlex
   # Project version
-  VERSION = "0.5.3"
+  VERSION = "0.6.0"
 end

data/rlex.gemspec

@@ -17,4 +17,6 @@ Gem::Specification.new do |gem|
   gem.name          = "rlex"
   gem.require_paths = ["lib"]
   gem.version       = Rlex::VERSION
+
+  gem.add_development_dependency "rspec"
 end

data/spec/rlex/lexer_spec.rb

@@ -32,10 +32,9 @@ describe Lexer do
       @lexer.ignore /\s+/
       @lexer.keyword :special
       @lexer.start " \t\nspecialspecial   special   "
-      special = Token.new :special, "special"
-      @lexer.next_token.should eq special
-      @lexer.next_token.should eq special
-      @lexer.next_token.should eq special
+      @lexer.next_token.should eq Token.new(:special, "special", 2, 0)
+      @lexer.next_token.should eq Token.new(:special, "special", 2, 7)
+      @lexer.next_token.should eq Token.new(:special, "special", 2, 17)
       @lexer.next_token.should eq EOF_TOKEN
     end
 
@@ -43,10 +42,10 @@ describe Lexer do
       @lexer.ignore /\s+/
       @lexer.rule :word, /\w+/
       @lexer.start "sentence with four tokens"
-      @lexer.next_token.should eq Token.new :word, "sentence"
-      @lexer.next_token.should eq Token.new :word, "with"
-      @lexer.next_token.should eq Token.new :word, "four"
-      @lexer.next_token.should eq Token.new :word, "tokens"
+      @lexer.next_token.should eq Token.new(:word, "sentence", 1, 0)
+      @lexer.next_token.should eq Token.new(:word, "with", 1, 9)
+      @lexer.next_token.should eq Token.new(:word, "four", 1, 14)
+      @lexer.next_token.should eq Token.new(:word, "tokens", 1, 19)
       @lexer.next_token.should eq EOF_TOKEN
     end
 
@@ -57,11 +56,22 @@ describe Lexer do
       @lexer.keyword :rparen, ")"
       @lexer.rule :word, /\w+/
       @lexer.start "ifu ( if ) ifu"
-      @lexer.next_token.should eq Token.new :word, "ifu"
-      @lexer.next_token.should eq Token.new :lparen, "("
-      @lexer.next_token.should eq Token.new :if, "if"
-      @lexer.next_token.should eq Token.new :rparen, ")"
-      @lexer.next_token.should eq Token.new :word, "ifu"
+      @lexer.next_token.should eq Token.new(:word, "ifu", 1, 0)
+      @lexer.next_token.should eq Token.new(:lparen, "(", 1, 4)
+      @lexer.next_token.should eq Token.new(:if, "if", 1, 6)
+      @lexer.next_token.should eq Token.new(:rparen, ")", 1, 9)
+      @lexer.next_token.should eq Token.new(:word, "ifu", 1, 11)
+      @lexer.next_token.should eq EOF_TOKEN
+    end
+
+    it "should recognize keywords even if declared after rules which also match" do
+      @lexer.ignore /\s+/
+      @lexer.rule :word, /\w+/
+      @lexer.keyword :keyword
+      @lexer.start "word keyword keywordmore"
+      @lexer.next_token.should eq Token.new(:word, "word", 1, 0)
+      @lexer.next_token.should eq Token.new(:keyword, "keyword", 1, 5)
+      @lexer.next_token.should eq Token.new(:word, "keywordmore", 1, 13)
       @lexer.next_token.should eq EOF_TOKEN
     end
   end

metadata

@@ -2,7 +2,7 @@
 name: rlex
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.5.3
+  version: 0.6.0
 platform: ruby
 authors: 
 - Rasmus Borgsmidt
@@ -10,9 +10,19 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-04-25 00:00:00 Z
-dependencies: []
-
+date: 2012-05-16 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id001
 description: Implements a simple lexer using a StringScanner
 email: 
 - rasmus@borgsmidt.dk
@@ -26,6 +36,7 @@ files:
 - .gitignore
 - .rspec
 - .yardopts
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md