rubylexer 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/tokenprinter.rb ADDED
@@ -0,0 +1,152 @@
1
+ =begin copyright
2
+ rubylexer - a ruby lexer written in ruby
3
+ Copyright (C) 2004,2005 Caleb Clausen
4
+
5
+ This library is free software; you can redistribute it and/or
6
+ modify it under the terms of the GNU Lesser General Public
7
+ License as published by the Free Software Foundation; either
8
+ version 2.1 of the License, or (at your option) any later version.
9
+
10
+ This library is distributed in the hope that it will be useful,
11
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13
+ Lesser General Public License for more details.
14
+
15
+ You should have received a copy of the GNU Lesser General Public
16
+ License along with this library; if not, write to the Free Software
17
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
+ =end
19
+
20
+
21
+
22
+ require "assert"
23
+
24
+
25
+
26
+
27
+ #-------------------------------
28
+ class SimpleTokenPrinter
29
+
30
+ def initialize(*bogus)
31
+ @tokens=@srclines=0
32
+ end
33
+
34
+ TOKENSPERLINE=8
35
+ TOKENSMAGICMAP="\n"+' '*(TOKENSPERLINE-1)
36
+
37
+ def pprint(tok) print(sprint(tok)) end
38
+
39
+ def sprint(tok)
40
+ case tok
41
+ when Newline then "#{@lastfal if ((@srclines+=1)%16==0)} #; "
42
+ when FileAndLineToken then @lastfal=tok;''
43
+ when IgnoreToken then '' #skip comments&whitespace
44
+ else tok.to_s + TOKENSMAGICMAP[ (@tokens+=1) % TOKENSPERLINE, 1 ]
45
+ end
46
+ end
47
+ end
48
+
49
+ class EscNl; def ws_munge(tp)
50
+ tp.lasttok=self
51
+ return to_s
52
+ end end
53
+ class FileAndLineToken; def ws_munge(tp)
54
+ tp.lasttok=self
55
+ tp.lastfal=self
56
+ return ''
57
+ end end
58
+ class Newline; def ws_munge(tp)
59
+ tp.lasttok=self
60
+ return"#{tp.lastfal_every_10_lines}\n"
61
+ end end
62
+ class IgnoreToken; def ws_munge(tp)
63
+ #tp.latestline+= to_s.scan("\n").size
64
+ tp.lasttok=self
65
+ unless tp.inws
66
+ tp.inws=true
67
+ return ' '
68
+ end
69
+ return ''
70
+ end end
71
+ class OutlinedHereBodyToken; def ws_munge(tp)
72
+ nil
73
+ end end
74
+ class ZwToken; def ws_munge(tp)
75
+ case tp.showzw
76
+ when 2: explicit_form_all
77
+ when 1: explicit_form
78
+ when 0: nil
79
+ else raise 'unknown showzw'
80
+ end
81
+ end end
82
+ class Token; def ws_munge(tp)
83
+ nil
84
+ end end
85
+
86
+ #-------------------------------
87
+ class KeepWsTokenPrinter
88
+ attr_accessor :lasttok, :lastfal, :inws #,:latestline
89
+ attr :showzw
90
+ ACCUMSIZE=50
91
+
92
+
93
+ def initialize(sep='',line=1,showzw=0)
94
+ @sep=sep
95
+ @inws=false
96
+ @lasttok=''
97
+ #@latestline=line
98
+ @lastprintline=0
99
+ @accum=[]
100
+ @showzw=showzw
101
+ end
102
+
103
+ def pprint(tok)
104
+
105
+ @accum<<aprint(tok).to_s
106
+ if @accum.size>ACCUMSIZE or EoiToken===tok
107
+ print(@accum)
108
+ @accum=[]
109
+ end
110
+ end
111
+
112
+ def aprint(tok)
113
+ result=tok.ws_munge(self) and return result
114
+
115
+ #insert extra ws unless an ambiguous op immediately follows
116
+ #id or num, in which case ws would change the meaning
117
+ result=if (ZwToken===tok or NoWsToken===@lasttok)
118
+ tok.to_s
119
+ else
120
+ [@sep.dup,tok.to_s]
121
+ end
122
+
123
+ @lasttok=tok
124
+ @inws=false
125
+
126
+ return result
127
+ end
128
+
129
+ alias sprint aprint
130
+
131
+ def lastfal_every_10_lines
132
+ if(@lastprintline+=1) > 10
133
+ @lastprintline=0
134
+ %Q[ #@lastfal]
135
+ end
136
+ end
137
+
138
+
139
+ #insert extra ws unless an ambiguous op immediately follows
140
+ #id or num, in which case ws would change the meaning
141
+ def ws_would_change_meaning?(tok,lasttok) #yukk -- is it used?
142
+ tok.kind_of?(String) and
143
+ /^[%(]$/===tok and #is ambiguous operator?
144
+ lasttok and
145
+ (lasttok.kind_of?(Numeric) or
146
+ (lasttok.kind_of?(String) and
147
+ /^[$@a-zA-Z_]/===@lasttok)) #lasttok is id or num?
148
+ end
149
+ end
150
+
151
+ #-------------------------------
152
+
metadata ADDED
@@ -0,0 +1,76 @@
1
+ --- !ruby/object:Gem::Specification
2
+ rubygems_version: 0.8.10
3
+ specification_version: 1
4
+ name: rubylexer
5
+ version: !ruby/object:Gem::Version
6
+ version: 0.6.2
7
+ date: 2005-06-03
8
+ summary: lexer of ruby in ruby
9
+ require_paths:
10
+ - "."
11
+ email:
12
+ homepage:
13
+ rubyforge_project: rubylexer
14
+ description:
15
+ autorequire:
16
+ default_executable:
17
+ bindir: bin
18
+ has_rdoc: false
19
+ required_ruby_version: !ruby/object:Gem::Version::Requirement
20
+ requirements:
21
+ -
22
+ - ">"
23
+ - !ruby/object:Gem::Version
24
+ version: 0.0.0
25
+ version:
26
+ platform: ruby
27
+ authors:
28
+ - Caleb Clausen
29
+ files:
30
+ - rubylexer.rb
31
+ - assert.rb
32
+ - context.rb
33
+ - rlold.rb
34
+ - rulexer.rb
35
+ - tokenprinter.rb
36
+ - charhandler.rb
37
+ - io.each_til_charset.rb
38
+ - rubycode.rb
39
+ - symboltable.rb
40
+ - charset.rb
41
+ - require.rb
42
+ - token.rb
43
+ - COPYING
44
+ - README
45
+ - Rantfile
46
+ - howtouse.txt
47
+ - testing.txt
48
+ - testresults/placeholder
49
+ - testcode/deletewarns.rb
50
+ - testcode/dumptokens.rb
51
+ - testcode/locatetest
52
+ - testcode/rubylexervsruby.rb
53
+ - testcode/rubylexervsruby.sh
54
+ - testcode/tokentest.rb
55
+ - testcode/torment
56
+ - testdata/1.rb.broken
57
+ - testdata/23.rb
58
+ - testdata/g.rb
59
+ - testdata/newsyntax.rb
60
+ - testdata/noeolatend.rb
61
+ - testdata/p.rb
62
+ - testdata/pleac.rb.broken
63
+ - testdata/pre.rb
64
+ - testdata/pre.unix.rb
65
+ - testdata/regtest.rb
66
+ - testdata/tokentest.assert.rb.can
67
+ - testdata/untitled1.rb
68
+ - testdata/w.rb
69
+ - testdata/wsdlDriver.rb
70
+ test_files: []
71
+ rdoc_options: []
72
+ extra_rdoc_files: []
73
+ executables: []
74
+ extensions: []
75
+ requirements: []
76
+ dependencies: []