lexeme 0.0.2 → 0.0.3

data/lib/lexeme.rb

@@ -3,10 +3,9 @@ require 'lexeme/ruleset'
 require 'lexeme/token'
 require 'lexeme/lexeme'
 require 'lexeme/core_extensions'
+require 'lexeme/version'
 
 module Lexeme
-  VERSION = '0.0.2'
-  
   def self.analyze(source = nil)
     raise RuntimeError, 'Please use #define before calling #analyze.' unless @lexer
     
@@ -20,7 +19,7 @@ module Lexeme
   end
   
   def self.define(&block)
-    @lexer = Lexeme.new #unless @lexer
+    @lexer = Lexeme.new 
     @lexer.instance_eval(&block)
     
     @lexer

data/lib/lexeme/lexeme.rb

@@ -38,71 +38,59 @@ module Lexeme
     end
 
     private 
-
+    
+    # TODO: Work on the time complexity for this one
+    #       This could be better.
     def scan(input)
       previous = ''
       current  = ''
       tokens   = []
       line     = 1
-      string_state = false
 
       input.each_char do |c|
-        line += 1 if c == "\n"
-        
-        if c == "'" || c == '"'
-          previous << c
-          string_state ^= true
-          next
+        if c == "\n"
+          line += 1
+          c = ' '
         end
-
-        if string_state
-          previous << c
-          next
-        end
-
-        if ignorable?(c)
-          unless previous.empty?
-            token   = identify(previous)
-            raise RuntimeError, "Unknown token #{previous} on line #{line}!" if 
-              token.nil? || token.name.nil?
-
-            tokens << token 
-          end
-
+        
+        if !previous.empty? && ignorable?(previous)
           previous = ''
           current  = ''
-          next
         end
-
+  
         current << c
+        
         if !identifiable?(current)
-          raise RuntimeError, "Unknown token #{current} on line #{line}!" if 
-            previous.empty? 
+          raise RuntimeError, "Unknown token `#{current}` on line #{line}" if 
+            previous.empty?
           
           token = identify(previous)
-
-          raise RuntimeError, "Unknown token #{previous} on line #{line}!" if 
+          
+          raise RuntimeError, "Unknown token `#{previous}` on line #{line}" if 
             token.nil? || token.name.nil?
 
-          tokens << token
+          tokens  << token
           previous = c.clone
           current  = c.clone
-          
           next
         end
         
         previous = current.clone
       end
       
-      unless previous.empty?
-        tokens << identify(previous)
+      if !previous.empty? && !ignorable?(previous)
+        token = identify(previous)
+        raise RuntimeError, "Unknow token `#{previous}` on line #{line}" if
+          token.nil? || token.name.nil?
+        
+        tokens << token
       end
 
       tokens
     end
     
-    def ignorable?(char)
-      @ruleset.ignorable? char
+    def ignorable?(string)
+      @ruleset.ignorable?(string)
     end
     
     def identifiable?(string)

data/lib/lexeme/ruleset.rb

@@ -2,10 +2,15 @@ module Lexeme
   class Ruleset
     def initialize(&block)
       @rules  = []
-      @ignore = []
+      @ignore = [] 
       
+      # this is here to capture any other
+      # symbols that could be identified
+      # as var names, function names ...
       @unknown = Rule.new(nil, /^\w+$/)
-      @ignore << /\s+/
+
+      # this skips all whitespaces by default
+      @ignore << /^\s+/
 
       yield self if block_given?
     end
@@ -18,9 +23,9 @@ module Lexeme
       @ignore << regex
     end
 
-    def ignorable?(char)
+    def ignorable?(string)
       @ignore.each do |i|
-        return true if char =~ i
+        return true if string =~ i
       end
 
       false
@@ -32,6 +37,10 @@ module Lexeme
       end
 
       return true if string =~ @unknown.regex
+      
+      @ignore.each do |i|
+        return true if string =~ i
+      end
 
       false
     end

data/lib/lexeme/token.rb

@@ -7,12 +7,16 @@ module Lexeme
       @value = value
     end
 
-    def to_text
+    def to_s
       "#{@name}: #{@value}"
     end
 
-    def to_array
+    def to_ary
       [@name, @value]
     end
+
+    def to_array
+      to_ary
+    end
   end
 end

data/lib/lexeme/version.rb

@@ -0,0 +1,3 @@
+module Lexeme
+  VERSION = '0.0.3'
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lexeme
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-11 00:00:00.000000000 Z
+date: 2013-07-13 00:00:00.000000000 Z
 dependencies: []
 description: A simple lexical analyzer written in Ruby
 email: vladimir.ivic@icloud.com
@@ -23,6 +23,7 @@ files:
 - lib/lexeme/ruleset.rb
 - lib/lexeme/rule.rb
 - lib/lexeme/core_extensions.rb
+- lib/lexeme/version.rb
 homepage: http://rubygems.org/gems/lexeme
 licenses: []
 post_install_message: