lexeme 0.0.1

data/lib/lexeme/lexeme.rb

@@ -0,0 +1,81 @@
+module Lexeme
+  class Lexeme
+    attr_accessor :ruleset
+
+    def analyze(source)
+      raise ArgumentError, 'Argument 1 must be a String' unless
+        source.instance_of? String
+
+      raise ArgumentError, 'Source not defined' if 
+        source.empty?
+      
+      raise RuntimeError, 'Source file not readable' unless 
+        File.exists?(source)
+      
+      content = IO.read(source)
+      tokens  = scan(content)
+      
+      tokens
+    end
+
+    private 
+
+    def scan(input)
+      previous = ''
+      current  = ''
+      tokens   = []
+      line     = 1
+
+      input.each_char do |c|
+        line += 1 if c == "\n"
+
+        if ignorable?(c)
+          unless previous.empty?
+            token   = identify(previous)
+            raise RuntimeError, "Unknown token #{previous} on line #{line}!" if 
+              token.nil? || token.name.nil?
+
+            tokens << token 
+          end
+
+          previous = ''
+          current  = ''
+          next
+        end
+
+        current << c
+        if !identifiable?(current)
+          raise RuntimeError, "Unknown token #{current} on line #{line}!" if 
+            previous.empty? 
+          
+          token = identify(previous)
+
+          raise RuntimeError, "Unknown token #{previous} on line #{line}!" if 
+            token.nil? || token.name.nil?
+
+          tokens << token
+          previous = c.clone
+          current  = c.clone
+          
+          next
+        end
+        
+        previous = current.clone
+      end
+
+      tokens
+    end
+    
+    def ignorable?(char)
+      @ruleset.ignorable? char
+    end
+    
+    def identifiable?(string)
+      @ruleset.identifiable? string
+    end
+
+    def identify(string)
+      @ruleset.identify string
+    end
+  end
+end

data/lib/lexeme/rule.rb

@@ -0,0 +1,15 @@
+module Lexeme
+  class Rule
+    attr_reader :name, :regex
+
+    def initialize(name, regex)
+      raise ArgumentError, 'name must be a String' unless 
+        name.nil? || name.is_a?(String)
+      raise ArgumentError, 'regex must be a Regex' unless 
+        regex.is_a? Regexp
+      
+      @name  = name
+      @regex = regex
+    end
+  end
+end

data/lib/lexeme/ruleset.rb

@@ -0,0 +1,50 @@
+module Lexeme
+  class Ruleset
+    def initialize(&block)
+      @rules  = []
+      @ignore = []
+      
+      @unknown = Rule.new(nil, /^\w+$/)
+      @ignore << /\s+/
+
+      yield self if block_given?
+    end
+
+    def rule(name, regex)
+      @rules << Rule.new(name, regex)
+    end
+
+    def ignore(regex)
+      @ignore << regex
+    end
+
+    def ignorable?(char)
+      @ignore.each do |i|
+        return true if char =~ i
+      end
+
+      false
+    end
+
+    def identifiable?(string)
+      @rules.each do |r|
+        return true if string =~ r.regex
+      end
+
+      return true if string =~ @unknown.regex
+
+      false
+    end
+
+    def identify(string)
+      @rules.each do |r|
+        return Token.new(r.name, string) if string =~ r.regex
+      end
+      
+      return Token.new(@unknown.name, string) if 
+        string =~ @unknown.regex
+
+      nil
+    end
+  end 
+end

data/lib/lexeme/token.rb

@@ -0,0 +1,18 @@
+module Lexeme
+  class Token 
+    attr_reader :name, :value
+
+    def initialize(name, value)
+      @name  = name
+      @value = value
+    end
+
+    def to_text
+      "#{@name}: #{@value}"
+    end
+
+    def to_array
+      [@name, @value]
+    end
+  end
+end

data/lib/lexeme.rb

@@ -0,0 +1,20 @@
+require 'lexeme/rule'
+require 'lexeme/ruleset'
+require 'lexeme/token'
+require 'lexeme/lexeme'
+
+module Lexeme
+  VERSION = '0.0.1'
+
+  def self.setup
+    @lexer = Lexeme.new unless @lexer
+    yield @lexer if block_given?
+  end
+
+  def self.analyze(source)
+    tokens = @lexer.analyze(source)
+    tokens.each { |t| yield t } if block_given?
+
+    tokens
+  end
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: lexeme
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Vladimir Ivic
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-23 00:00:00.000000000 Z
+dependencies: []
+description: A smiple lexical analyzer written in Ruby
+email: vladimir.ivic@me.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/lexeme.rb
+- lib/lexeme/lexeme.rb
+- lib/lexeme/token.rb
+- lib/lexeme/ruleset.rb
+- lib/lexeme/rule.rb
+homepage: http://rubygems.org/gems/lexeme
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Lexeme
+test_files: []