lexr 0.1.0

data/README.md

@@ -0,0 +1,42 @@
+## Lexr
+
+Lexr is a lightweight lexical analyser written in ruby, it has no dependencies, has good test coverage, looks pretty and reads well.
+
+# An example: Expressions
+
+	ExpressionLexer = Lexr.that {
+		ignores /\s+/ => :whitespace
+		matches /[-+]?[0-9]*\.?[0-9]+/ => :number, :convert_with => lambda { |v| Float(v) }
+		matches "+" => :addition
+		matches "-" => :subtraction
+		matches "*" => :multiplication
+		matches "/" => :division
+		matches "(" => :left_parenthesis
+		matches ")" => :right_parenthesis
+	}
+
+	lexer = ExpressionLexer.new("1 * 12.5 / (55 + 2 - 56)")
+
+	while (token = lexer.next) != Lexr::Token.end
+		puts token
+	end
+
+results in an output of
+
+	number(1.0)
+	multiplication(*)
+	number(12.5)
+	division(/)
+	left_parenthesis(()
+	number(55.0)
+	addition(+)
+	number(2.0)
+	subtraction(-)
+	number(56.0)
+	right_parenthesis())
+	
+if you added a % in there somewhere, you'd get a Lexr::UnmatchableTextError with a message like this:
+
+	=> Unexpected character '%' at position 5
+	
+and that pretty is every feature so far. Please let me know of any bugs or additions that you'd like to see!

data/lexr.rb

@@ -0,0 +1 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'lib', 'lexr'))

data/lib/lexr.rb

@@ -0,0 +1,117 @@
+class Lexr
+	def self.that(&block)
+		dsl = Lexr::Dsl.new
+		block.arity == 1 ? block[dsl] : dsl.instance_eval(&block)
+		dsl
+	end
+	
+	def initialize(text, rules)
+		@text, @rules = text, rules
+		@position = 0
+	end
+
+	def next
+		return Lexr::Token.end if @position >= @text.length
+		@rules.each do |rule|
+			if result = rule.pattern.instance_of?(Regexp) ? regexp_match(rule.pattern) : literal_match(rule.pattern)
+				result = rule.converter[result] if rule.converter
+				return self.send(:next) if rule.ignore?
+				return Lexr::Token.new(result, rule.symbol)
+			end
+		end
+		raise Lexr::UnmatchableTextError.new(unprocessed_text[0..0], @position)
+	end
+	
+	private
+	
+	def unprocessed_text
+		@text[@position..-1]
+	end
+	
+	def regexp_match(regex)
+		return nil unless m = unprocessed_text.match(/^#{regex}/)
+		@position += m.end(0)
+		m[0]
+	end
+	
+	def literal_match(lit)
+		return nil unless unprocessed_text[0..lit.length-1] == lit
+		@position += lit.length
+		lit
+	end
+	
+	class Token
+		attr_reader :value, :type
+	
+		def initialize(value, type = nil)
+			@value, @type = value, type
+		end
+		
+		def self.method_missing(sym, *args)
+			self.new(args.first, sym)
+		end
+		
+		def to_s
+			"#{type}(#{value})"
+		end
+		
+		def ==(other)
+			@type == other.type && @value == other.value
+		end
+	end
+	
+	class Rule
+		attr_reader :pattern, :symbol
+		
+		def converter ; @opts[:convert_with] ; end
+		def ignore? ; @opts[:ignore] ; end
+		
+		def initialize(pattern, symbol, opts = {})
+			@pattern, @symbol, @opts = pattern, symbol, opts
+		end
+		
+		def ==(other)
+			@pattern == other.pattern && 
+				@symbol == other.symbol && 
+				@opts[:convert_with] == other.converter && 
+				@opts[:ignore] == other.ignore?
+		end
+	end
+	
+	class Dsl
+		def initialize
+			@rules = []
+		end
+		
+		def matches(rule_hash)
+			pattern = rule_hash.keys.reject { |k| k.class == Symbol }.first
+			symbol = rule_hash[pattern]
+			opts = rule_hash.delete_if { |k, v| k.class != Symbol }
+			@rules << Rule.new(pattern, symbol, opts)
+		end
+		
+		def ignores(rule_hash)
+			matches rule_hash.merge(:ignore => true)
+		end
+		
+		def new(str)
+			Lexr.new(str, @rules)
+		end
+	end
+	
+	class UnmatchableTextError < StandardError
+		attr_reader :character, :position
+		
+		def initialize(character, position)
+			@character, @position = character, position
+		end
+		
+		def message
+			"Unexpected character '#{character}' at position #{position + 1}"
+		end
+		
+		def inspect
+			message
+		end
+	end
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification 
+name: lexr
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Michael Baldry
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-12 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+description: 
+email: michael.baldry@uswitch.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+files: 
+- lexr.rb
+- README.md
+- lib/lexr.rb
+has_rdoc: true
+homepage: http://www.forwardtechnology.co.uk
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.md
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A lightweight and pretty lexical analyser
+test_files: []
+