rubylexer 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/COPYING +510 -0
- data/README +134 -0
- data/Rantfile +37 -0
- data/assert.rb +31 -0
- data/charhandler.rb +84 -0
- data/charset.rb +76 -0
- data/context.rb +174 -0
- data/howtouse.txt +136 -0
- data/io.each_til_charset.rb +247 -0
- data/require.rb +103 -0
- data/rlold.rb +12 -0
- data/rubycode.rb +44 -0
- data/rubylexer.rb +1589 -0
- data/rulexer.rb +532 -0
- data/symboltable.rb +65 -0
- data/testcode/deletewarns.rb +39 -0
- data/testcode/dumptokens.rb +38 -0
- data/testcode/locatetest +12 -0
- data/testcode/rubylexervsruby.rb +104 -0
- data/testcode/rubylexervsruby.sh +51 -0
- data/testcode/tokentest.rb +237 -0
- data/testcode/torment +51 -0
- data/testdata/1.rb.broken +729 -0
- data/testdata/23.rb +24 -0
- data/testdata/g.rb +15 -0
- data/testdata/newsyntax.rb +18 -0
- data/testdata/noeolatend.rb +1 -0
- data/testdata/p.rb +1227 -0
- data/testdata/pleac.rb.broken +6282 -0
- data/testdata/pre.rb +33 -0
- data/testdata/pre.unix.rb +33 -0
- data/testdata/regtest.rb +621 -0
- data/testdata/tokentest.assert.rb.can +7 -0
- data/testdata/untitled1.rb +1 -0
- data/testdata/w.rb +22 -0
- data/testdata/wsdlDriver.rb +499 -0
- data/testing.txt +130 -0
- data/testresults/placeholder +0 -0
- data/token.rb +486 -0
- data/tokenprinter.rb +152 -0
- metadata +76 -0
data/tokenprinter.rb
ADDED
@@ -0,0 +1,152 @@
|
|
1
|
+
=begin copyright
|
2
|
+
rubylexer - a ruby lexer written in ruby
|
3
|
+
Copyright (C) 2004,2005 Caleb Clausen
|
4
|
+
|
5
|
+
This library is free software; you can redistribute it and/or
|
6
|
+
modify it under the terms of the GNU Lesser General Public
|
7
|
+
License as published by the Free Software Foundation; either
|
8
|
+
version 2.1 of the License, or (at your option) any later version.
|
9
|
+
|
10
|
+
This library is distributed in the hope that it will be useful,
|
11
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
13
|
+
Lesser General Public License for more details.
|
14
|
+
|
15
|
+
You should have received a copy of the GNU Lesser General Public
|
16
|
+
License along with this library; if not, write to the Free Software
|
17
|
+
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
18
|
+
=end
|
19
|
+
|
20
|
+
|
21
|
+
|
22
|
+
require "assert"
|
23
|
+
|
24
|
+
|
25
|
+
|
26
|
+
|
27
|
+
#-------------------------------
|
28
|
+
class SimpleTokenPrinter
|
29
|
+
|
30
|
+
def initialize(*bogus)
|
31
|
+
@tokens=@srclines=0
|
32
|
+
end
|
33
|
+
|
34
|
+
TOKENSPERLINE=8
|
35
|
+
TOKENSMAGICMAP="\n"+' '*(TOKENSPERLINE-1)
|
36
|
+
|
37
|
+
def pprint(tok) print(sprint(tok)) end
|
38
|
+
|
39
|
+
def sprint(tok)
|
40
|
+
case tok
|
41
|
+
when Newline then "#{@lastfal if ((@srclines+=1)%16==0)} #; "
|
42
|
+
when FileAndLineToken then @lastfal=tok;''
|
43
|
+
when IgnoreToken then '' #skip comments&whitespace
|
44
|
+
else tok.to_s + TOKENSMAGICMAP[ (@tokens+=1) % TOKENSPERLINE, 1 ]
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
class EscNl; def ws_munge(tp)
|
50
|
+
tp.lasttok=self
|
51
|
+
return to_s
|
52
|
+
end end
|
53
|
+
class FileAndLineToken; def ws_munge(tp)
|
54
|
+
tp.lasttok=self
|
55
|
+
tp.lastfal=self
|
56
|
+
return ''
|
57
|
+
end end
|
58
|
+
class Newline; def ws_munge(tp)
|
59
|
+
tp.lasttok=self
|
60
|
+
return"#{tp.lastfal_every_10_lines}\n"
|
61
|
+
end end
|
62
|
+
class IgnoreToken; def ws_munge(tp)
|
63
|
+
#tp.latestline+= to_s.scan("\n").size
|
64
|
+
tp.lasttok=self
|
65
|
+
unless tp.inws
|
66
|
+
tp.inws=true
|
67
|
+
return ' '
|
68
|
+
end
|
69
|
+
return ''
|
70
|
+
end end
|
71
|
+
class OutlinedHereBodyToken; def ws_munge(tp)
|
72
|
+
nil
|
73
|
+
end end
|
74
|
+
class ZwToken; def ws_munge(tp)
|
75
|
+
case tp.showzw
|
76
|
+
when 2: explicit_form_all
|
77
|
+
when 1: explicit_form
|
78
|
+
when 0: nil
|
79
|
+
else raise 'unknown showzw'
|
80
|
+
end
|
81
|
+
end end
|
82
|
+
class Token; def ws_munge(tp)
|
83
|
+
nil
|
84
|
+
end end
|
85
|
+
|
86
|
+
#-------------------------------
|
87
|
+
class KeepWsTokenPrinter
|
88
|
+
attr_accessor :lasttok, :lastfal, :inws #,:latestline
|
89
|
+
attr :showzw
|
90
|
+
ACCUMSIZE=50
|
91
|
+
|
92
|
+
|
93
|
+
def initialize(sep='',line=1,showzw=0)
|
94
|
+
@sep=sep
|
95
|
+
@inws=false
|
96
|
+
@lasttok=''
|
97
|
+
#@latestline=line
|
98
|
+
@lastprintline=0
|
99
|
+
@accum=[]
|
100
|
+
@showzw=showzw
|
101
|
+
end
|
102
|
+
|
103
|
+
def pprint(tok)
|
104
|
+
|
105
|
+
@accum<<aprint(tok).to_s
|
106
|
+
if @accum.size>ACCUMSIZE or EoiToken===tok
|
107
|
+
print(@accum)
|
108
|
+
@accum=[]
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def aprint(tok)
|
113
|
+
result=tok.ws_munge(self) and return result
|
114
|
+
|
115
|
+
#insert extra ws unless an ambiguous op immediately follows
|
116
|
+
#id or num, in which case ws would change the meaning
|
117
|
+
result=if (ZwToken===tok or NoWsToken===@lasttok)
|
118
|
+
tok.to_s
|
119
|
+
else
|
120
|
+
[@sep.dup,tok.to_s]
|
121
|
+
end
|
122
|
+
|
123
|
+
@lasttok=tok
|
124
|
+
@inws=false
|
125
|
+
|
126
|
+
return result
|
127
|
+
end
|
128
|
+
|
129
|
+
alias sprint aprint
|
130
|
+
|
131
|
+
def lastfal_every_10_lines
|
132
|
+
if(@lastprintline+=1) > 10
|
133
|
+
@lastprintline=0
|
134
|
+
%Q[ #@lastfal]
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
|
139
|
+
#insert extra ws unless an ambiguous op immediately follows
|
140
|
+
#id or num, in which case ws would change the meaning
|
141
|
+
def ws_would_change_meaning?(tok,lasttok) #yukk -- is it used?
|
142
|
+
tok.kind_of?(String) and
|
143
|
+
/^[%(]$/===tok and #is ambiguous operator?
|
144
|
+
lasttok and
|
145
|
+
(lasttok.kind_of?(Numeric) or
|
146
|
+
(lasttok.kind_of?(String) and
|
147
|
+
/^[$@a-zA-Z_]/===@lasttok)) #lasttok is id or num?
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
#-------------------------------
|
152
|
+
|
metadata
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
rubygems_version: 0.8.10
|
3
|
+
specification_version: 1
|
4
|
+
name: rubylexer
|
5
|
+
version: !ruby/object:Gem::Version
|
6
|
+
version: 0.6.2
|
7
|
+
date: 2005-06-03
|
8
|
+
summary: lexer of ruby in ruby
|
9
|
+
require_paths:
|
10
|
+
- "."
|
11
|
+
email:
|
12
|
+
homepage:
|
13
|
+
rubyforge_project: rubylexer
|
14
|
+
description:
|
15
|
+
autorequire:
|
16
|
+
default_executable:
|
17
|
+
bindir: bin
|
18
|
+
has_rdoc: false
|
19
|
+
required_ruby_version: !ruby/object:Gem::Version::Requirement
|
20
|
+
requirements:
|
21
|
+
-
|
22
|
+
- ">"
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: 0.0.0
|
25
|
+
version:
|
26
|
+
platform: ruby
|
27
|
+
authors:
|
28
|
+
- Caleb Clausen
|
29
|
+
files:
|
30
|
+
- rubylexer.rb
|
31
|
+
- assert.rb
|
32
|
+
- context.rb
|
33
|
+
- rlold.rb
|
34
|
+
- rulexer.rb
|
35
|
+
- tokenprinter.rb
|
36
|
+
- charhandler.rb
|
37
|
+
- io.each_til_charset.rb
|
38
|
+
- rubycode.rb
|
39
|
+
- symboltable.rb
|
40
|
+
- charset.rb
|
41
|
+
- require.rb
|
42
|
+
- token.rb
|
43
|
+
- COPYING
|
44
|
+
- README
|
45
|
+
- Rantfile
|
46
|
+
- howtouse.txt
|
47
|
+
- testing.txt
|
48
|
+
- testresults/placeholder
|
49
|
+
- testcode/deletewarns.rb
|
50
|
+
- testcode/dumptokens.rb
|
51
|
+
- testcode/locatetest
|
52
|
+
- testcode/rubylexervsruby.rb
|
53
|
+
- testcode/rubylexervsruby.sh
|
54
|
+
- testcode/tokentest.rb
|
55
|
+
- testcode/torment
|
56
|
+
- testdata/1.rb.broken
|
57
|
+
- testdata/23.rb
|
58
|
+
- testdata/g.rb
|
59
|
+
- testdata/newsyntax.rb
|
60
|
+
- testdata/noeolatend.rb
|
61
|
+
- testdata/p.rb
|
62
|
+
- testdata/pleac.rb.broken
|
63
|
+
- testdata/pre.rb
|
64
|
+
- testdata/pre.unix.rb
|
65
|
+
- testdata/regtest.rb
|
66
|
+
- testdata/tokentest.assert.rb.can
|
67
|
+
- testdata/untitled1.rb
|
68
|
+
- testdata/w.rb
|
69
|
+
- testdata/wsdlDriver.rb
|
70
|
+
test_files: []
|
71
|
+
rdoc_options: []
|
72
|
+
extra_rdoc_files: []
|
73
|
+
executables: []
|
74
|
+
extensions: []
|
75
|
+
requirements: []
|
76
|
+
dependencies: []
|