simple_lexer 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in simple_lexer.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 William
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,29 @@
1
+ # SimpleLexer
2
+
3
+ TODO: Write a gem description
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'simple_lexer'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install simple_lexer
18
+
19
+ ## Usage
20
+
21
+ TODO: Write usage instructions here
22
+
23
+ ## Contributing
24
+
25
+ 1. Fork it
26
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
27
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
28
+ 4. Push to the branch (`git push origin my-new-feature`)
29
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,3 @@
1
+ module SimpleLexer
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,119 @@
1
+ require_relative "simple_lexer/version"
2
+
3
+ module SimpleLexer
4
+
5
+ class NoMatchError < Exception
6
+ # unable to match
7
+ end
8
+
9
+ class EndOfStreamException < Exception
10
+ # when the Lexer is finished
11
+ end
12
+
13
+ class Lexer
14
+
15
+ attr_reader :rules, :pos
16
+
17
+ def initialize(&rules)
18
+ @rules = [] # list of {:rule => Regexp, :token => :token_id}
19
+ @ignore = [] # list of Regexp
20
+ @pos = 0 # position in input
21
+ instance_eval &rules
22
+ end
23
+
24
+ def tok(rule, token, &action)
25
+ # defining a new rule:
26
+ #
27
+ # my_lexer = SimpleLexer::Lexer.new do
28
+ # tok /\w+/, :identifier
29
+ # end
30
+
31
+ @rules << {:rule => Regexp.new('\A' + rule.source), :token => token, :action => action}
32
+ end
33
+
34
+ def ign(rule)
35
+ # defining conditions to ignore:
36
+ #
37
+ # my_lexer = SimpleLexer::Lexer.new do
38
+ # tok /\w+/, :identifier
39
+ # ign :whitespace
40
+ # end
41
+
42
+ if rule == :whitespace
43
+ rule = /\s+/
44
+ end
45
+
46
+ @ignore << Regexp.new('\A' + rule.source)
47
+ end
48
+
49
+ def load=(string)
50
+ # load a string into the lexer
51
+ # my_lexer.load( ... )
52
+
53
+ @load = string
54
+ @pos = 0
55
+ end
56
+
57
+ def load
58
+ # what the lexer currently sees
59
+ # my_lexer.load ...
60
+
61
+ @load[@pos..-1]
62
+ end
63
+
64
+ def next_token
65
+ # get the next token
66
+ # my_lexer.next_token -> [ :token => :token_id, :text => matched ]
67
+ for rule in @ignore
68
+ if match = load[rule]
69
+ @pos += match.length
70
+ end
71
+ end
72
+
73
+ if @pos >= @load.length
74
+ raise EndOfStreamException, "Finished lexing, no more tokens left."
75
+ end
76
+
77
+ for rule in @rules
78
+ if match = load[rule[:rule]]
79
+ @pos += match.length
80
+ return {:token => rule[:token], :text => match,
81
+ :value => (!rule[:action].nil? ? rule[:action].call(match) : nil) }
82
+ end
83
+ end
84
+
85
+ raise NoMatchError, "Unable to match, unexpected characters: '#{load[0..10]}...'"
86
+ end
87
+
88
+ def all_tokens
89
+ # returns the array of all tokens until it is finished lexing
90
+ # my_lexer.all_tokens
91
+
92
+ tokens = []
93
+ loop do
94
+ tokens << next_token
95
+ end
96
+ rescue EndOfStreamException => e
97
+ tokens
98
+ end
99
+
100
+ def finished?
101
+ return @pos >= @load.length
102
+ end
103
+
104
+ end
105
+ end
106
+
107
+ my_lexer = SimpleLexer::Lexer.new do
108
+ tok /\+/, :plus
109
+ tok /-/, :minus
110
+ tok /\//, :div
111
+ tok /\*/, :mult
112
+ tok /\(/, :lparen
113
+ tok /\)/, :rparen
114
+ tok /-?\d+(\.\d+)?/, :number do |t| t.to_f end
115
+ ign :whitespace
116
+ end
117
+
118
+ my_lexer.load = "321.32 + 432.388 - 33/4.3 - 4.228 * 5 - (32*632)"
119
+ p my_lexer.all_tokens
@@ -0,0 +1,23 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'simple_lexer/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "simple_lexer"
8
+ spec.version = SimpleLexer::VERSION
9
+ spec.authors = ["William"]
10
+ spec.email = ["wchen298@gmail.com"]
11
+ spec.description = %q{A simple toy lexer for Ruby}
12
+ spec.summary = %q{Rudimentary lexer for Ruby}
13
+ spec.homepage = ""
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files`.split($/)
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_development_dependency "bundler", "~> 1.3"
22
+ spec.add_development_dependency "rake"
23
+ end
metadata ADDED
@@ -0,0 +1,76 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: simple_lexer
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - William
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-11-16 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: &12567780 !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: '1.3'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: *12567780
25
+ - !ruby/object:Gem::Dependency
26
+ name: rake
27
+ requirement: &12567100 !ruby/object:Gem::Requirement
28
+ none: false
29
+ requirements:
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
32
+ version: '0'
33
+ type: :development
34
+ prerelease: false
35
+ version_requirements: *12567100
36
+ description: A simple toy lexer for Ruby
37
+ email:
38
+ - wchen298@gmail.com
39
+ executables: []
40
+ extensions: []
41
+ extra_rdoc_files: []
42
+ files:
43
+ - .gitignore
44
+ - Gemfile
45
+ - LICENSE.txt
46
+ - README.md
47
+ - Rakefile
48
+ - lib/simple_lexer.rb
49
+ - lib/simple_lexer/version.rb
50
+ - simple_lexer.gemspec
51
+ homepage: ''
52
+ licenses:
53
+ - MIT
54
+ post_install_message:
55
+ rdoc_options: []
56
+ require_paths:
57
+ - lib
58
+ required_ruby_version: !ruby/object:Gem::Requirement
59
+ none: false
60
+ requirements:
61
+ - - ! '>='
62
+ - !ruby/object:Gem::Version
63
+ version: '0'
64
+ required_rubygems_version: !ruby/object:Gem::Requirement
65
+ none: false
66
+ requirements:
67
+ - - ! '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ requirements: []
71
+ rubyforge_project:
72
+ rubygems_version: 1.8.11
73
+ signing_key:
74
+ specification_version: 3
75
+ summary: Rudimentary lexer for Ruby
76
+ test_files: []