lexeme 0.0.4 → 0.0.5

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ODliZDY5NjQ5YTlmZmRjZTYyYjRhNjJlZmVlNWIxYTJiOTdmMWYyYw==
+  data.tar.gz: !binary |-
+    NjFhZWRlMjY0NmEwMDk2MjZiMGU1NWY1YWFmYWZjZDA5MzQ5NGJjMw==
+SHA512:
+  metadata.gz: !binary |-
+    YzE0YWJjYzY4OGQ3NThlM2FlYTQ0ODc0MjE3MWE4ZjFmYWM3Zjk4MmQ3MDA0
+    NTYyNDc3NzJmYjQxYmY1ZGM0YjM0OTFiYzIxZGVmOGFiMjViODIwZGI5Y2Ix
+    M2ZiOTQ0YTYzMzRjMmVlZDg1MWNkNzllMTRmNjI4YWFiNDc3NmE=
+  data.tar.gz: !binary |-
+    MzE4Mjk4NTIxZDkwYzkwOTE4ZjFmN2ExMDMyNjJlYWYxZWEzMDg4YjA0Nzhm
+    YTNjNjI5MzJiNGY3ZTI0ZDY4OGQwYTMyYTkwOTg5ODdiYzNmMzBmMzRhMmNi
+    OTdhYzY1OWRlMTdlN2Q0YmMwNTcxNzAzMzE4MjIwZTgzODM0NDQ=

data/lib/lexeme/core_extensions.rb

@@ -1,6 +1,5 @@
 class ::String
-  # Will move to tokenize and have #to_tokens obsolete
-  def to_tokens
+  def tokenize 
     @lexer ||= Lexeme.define do 
       use_language :natural
     end
@@ -14,7 +13,8 @@ class ::String
     @lexer.tokens
   end
 
-  def tokenize
-    to_tokens
+  def to_tokens
+    warn "String#to_tokens is obsolete, use String#tokenize instead"
+    tokenize
   end
 end

data/lib/lexeme/lexeme.rb

@@ -41,10 +41,12 @@ module Lexeme
     end
     
     def use_language(name)
-      require "lexeme/languages/#{name}.rb"
+      lang_file = "lexeme/languages/#{name}.rb"
+      require lang_file
+
       instance_eval(&Language::send(name))
     rescue LoadError
-      abort "Language file cannot be found"
+      abort "Language file #{lang_file} cannot be found"
     end
 
     private 
@@ -55,12 +57,16 @@ module Lexeme
       previous = ''
       current  = ''
       tokens   = []
-      line     = 1
+      new_line = 0
+      line_num = 1
 
       input.each_char do |c|
         if c == "\n"
-          line += 1
+          new_line += 1
+          line_num += 1
           c = ' '
+        else 
+          new_line = 0
         end
         
         if !previous.empty? && ignorable?(previous)
@@ -74,7 +80,7 @@ module Lexeme
           raise RuntimeError, "Unknown token `#{current}` on line #{line}" if 
             previous.empty?
           
-          token = identify(previous)
+          token = identify(previous, line_num - new_line)
           
           raise RuntimeError, "Unknown token `#{previous}` on line #{line}" if 
             token.nil? || token.name.nil?
@@ -89,7 +95,7 @@ module Lexeme
       end
       
       if !previous.empty? && !ignorable?(previous)
-        token = identify(previous)
+        token = identify(previous, line_num - new_line) 
         raise RuntimeError, "Unknow token `#{previous}` on line #{line}" if
           token.nil? || token.name.nil?
         
@@ -104,11 +110,11 @@ module Lexeme
     end
     
     def identifiable?(string)
-      @ruleset.identifiable? string
+      @ruleset.identifiable?(string)
     end
 
-    def identify(string)
-      @ruleset.identify string
+    def identify(string, line)
+      @ruleset.identify(string, line)
     end
   end
 end

data/lib/lexeme/ruleset.rb

@@ -45,12 +45,12 @@ module Lexeme
       false
     end
 
-    def identify(string)
+    def identify(string, line)
       @rules.each do |r|
-        return Token.new(r.name, string) if string =~ r.regex
+        return Token.new(r.name, string, line) if string =~ r.regex
       end
       
-      return Token.new(@unknown.name, string) if 
+      return Token.new(@unknown.name, string, line) if 
         string =~ @unknown.regex
 
       nil

data/lib/lexeme/token.rb

@@ -1,22 +1,31 @@
 module Lexeme
   class Token 
-    attr_reader :name, :value
+    attr_reader :name, :value, :line
 
-    def initialize(name, value)
+    def initialize(name, value, line)
       @name  = name
       @value = value
+      @line  = line
     end
 
     def to_s
-      "#{@name}: #{@value}"
+      "#{line} => #{name} : #{value}"
+    end
+
+    def to_hash
+      {:line => line, :name => name, :value => value}
     end
 
     def to_ary
-      [@name, @value]
+      [name, value, line]
     end
 
     def to_array
       to_ary
     end
+
+    def to_a
+      to_ary
+    end
   end
 end

data/lib/lexeme/version.rb

@@ -1,3 +1,3 @@
 module Lexeme
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lexeme
 version: !ruby/object:Gem::Version
-  version: 0.0.4
-  prerelease: 
+  version: 0.0.5
 platform: ruby
 authors:
 - Vladimir Ivic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-15 00:00:00.000000000 Z
+date: 2014-03-25 00:00:00.000000000 Z
 dependencies: []
 description: A simple lexical analyzer written in Ruby
 email: vladimir.ivic@icloud.com
@@ -26,27 +25,27 @@ files:
 - lib/lexeme/version.rb
 - lib/lexeme/languages/natural.rb
 homepage: https://github.com/mancmelou/lexeme
-licenses: []
+licenses:
+- MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.1.4
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Lexeme
 test_files: []