sqlpostgres 1.2.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (207) hide show
  1. data/Gemfile +8 -0
  2. data/Gemfile.lock +22 -0
  3. data/LICENSE.md +23 -0
  4. data/README.rdoc +59 -0
  5. data/Rakefile +32 -0
  6. data/VERSION +1 -0
  7. data/doc/BUGS +2 -0
  8. data/doc/examples/README +6 -0
  9. data/doc/examples/connection.rb +16 -0
  10. data/doc/examples/connection_auto.rb +22 -0
  11. data/doc/examples/connection_ctor.rb +18 -0
  12. data/doc/examples/connection_default.rb +15 -0
  13. data/doc/examples/connection_exec.rb +18 -0
  14. data/doc/examples/connection_manual.rb +12 -0
  15. data/doc/examples/connection_wrapped_new.rb +13 -0
  16. data/doc/examples/connection_wrapped_open.rb +13 -0
  17. data/doc/examples/cursor.rb +38 -0
  18. data/doc/examples/include_module.rb +9 -0
  19. data/doc/examples/include_module2.rb +12 -0
  20. data/doc/examples/insert.rb +30 -0
  21. data/doc/examples/insert2.rb +36 -0
  22. data/doc/examples/insert_bytea.rb +16 -0
  23. data/doc/examples/insert_bytea_array.rb +17 -0
  24. data/doc/examples/insert_default_values.rb +16 -0
  25. data/doc/examples/insert_insert.rb +16 -0
  26. data/doc/examples/insert_insert_default.rb +16 -0
  27. data/doc/examples/insert_insert_select.rb +20 -0
  28. data/doc/examples/insert_select.rb +20 -0
  29. data/doc/examples/interval.rb +17 -0
  30. data/doc/examples/savepoint.rb +38 -0
  31. data/doc/examples/select.rb +33 -0
  32. data/doc/examples/select2.rb +36 -0
  33. data/doc/examples/select_cross_join.rb +18 -0
  34. data/doc/examples/select_distinct.rb +18 -0
  35. data/doc/examples/select_distinct_on +19 -0
  36. data/doc/examples/select_for_update.rb +18 -0
  37. data/doc/examples/select_from.rb +17 -0
  38. data/doc/examples/select_from_subselect.rb +20 -0
  39. data/doc/examples/select_group_by.rb +19 -0
  40. data/doc/examples/select_having.rb +20 -0
  41. data/doc/examples/select_join_on.rb +18 -0
  42. data/doc/examples/select_join_using.rb +18 -0
  43. data/doc/examples/select_limit.rb +19 -0
  44. data/doc/examples/select_natural_join.rb +18 -0
  45. data/doc/examples/select_offset.rb +19 -0
  46. data/doc/examples/select_order_by.rb +20 -0
  47. data/doc/examples/select_select.rb +30 -0
  48. data/doc/examples/select_select_alias.rb +30 -0
  49. data/doc/examples/select_select_expression.rb +31 -0
  50. data/doc/examples/select_select_literal.rb +24 -0
  51. data/doc/examples/select_union.rb +21 -0
  52. data/doc/examples/select_where_array.rb +18 -0
  53. data/doc/examples/select_where_in.rb +18 -0
  54. data/doc/examples/select_where_string.rb +18 -0
  55. data/doc/examples/simple.rb +34 -0
  56. data/doc/examples/transaction.rb +30 -0
  57. data/doc/examples/transaction_abort.rb +30 -0
  58. data/doc/examples/transaction_commit.rb +34 -0
  59. data/doc/examples/translate_substitute_values.rb +17 -0
  60. data/doc/examples/update.rb +32 -0
  61. data/doc/examples/update2.rb +44 -0
  62. data/doc/examples/update_only.rb +17 -0
  63. data/doc/examples/update_set.rb +17 -0
  64. data/doc/examples/update_set_array.rb +16 -0
  65. data/doc/examples/update_set_bytea.rb +16 -0
  66. data/doc/examples/update_set_expression.rb +16 -0
  67. data/doc/examples/update_set_subselect.rb +20 -0
  68. data/doc/examples/update_where.rb +17 -0
  69. data/doc/examples/use_prefix.rb +8 -0
  70. data/doc/examples/use_prefix2.rb +11 -0
  71. data/doc/index.html +31 -0
  72. data/doc/insertexamples.rb +9 -0
  73. data/doc/makemanual +4 -0
  74. data/doc/makerdoc +5 -0
  75. data/doc/manual.dbk +622 -0
  76. data/lib/sqlpostgres/Connection.rb +198 -0
  77. data/lib/sqlpostgres/Cursor.rb +157 -0
  78. data/lib/sqlpostgres/Delete.rb +67 -0
  79. data/lib/sqlpostgres/Exceptions.rb +15 -0
  80. data/lib/sqlpostgres/Insert.rb +279 -0
  81. data/lib/sqlpostgres/NullConnection.rb +22 -0
  82. data/lib/sqlpostgres/PgBit.rb +73 -0
  83. data/lib/sqlpostgres/PgBox.rb +37 -0
  84. data/lib/sqlpostgres/PgCidr.rb +21 -0
  85. data/lib/sqlpostgres/PgCircle.rb +75 -0
  86. data/lib/sqlpostgres/PgInet.rb +21 -0
  87. data/lib/sqlpostgres/PgInterval.rb +208 -0
  88. data/lib/sqlpostgres/PgLineSegment.rb +37 -0
  89. data/lib/sqlpostgres/PgMacAddr.rb +21 -0
  90. data/lib/sqlpostgres/PgPath.rb +64 -0
  91. data/lib/sqlpostgres/PgPoint.rb +65 -0
  92. data/lib/sqlpostgres/PgPolygon.rb +56 -0
  93. data/lib/sqlpostgres/PgTime.rb +77 -0
  94. data/lib/sqlpostgres/PgTimeWithTimeZone.rb +98 -0
  95. data/lib/sqlpostgres/PgTimestamp.rb +93 -0
  96. data/lib/sqlpostgres/PgTwoPoints.rb +54 -0
  97. data/lib/sqlpostgres/PgType.rb +34 -0
  98. data/lib/sqlpostgres/PgWrapper.rb +41 -0
  99. data/lib/sqlpostgres/Savepoint.rb +98 -0
  100. data/lib/sqlpostgres/Select.rb +855 -0
  101. data/lib/sqlpostgres/Transaction.rb +120 -0
  102. data/lib/sqlpostgres/Translate.rb +436 -0
  103. data/lib/sqlpostgres/Update.rb +188 -0
  104. data/lib/sqlpostgres.rb +67 -0
  105. data/test/Assert.rb +72 -0
  106. data/test/Connection.test.rb +246 -0
  107. data/test/Cursor.test.rb +190 -0
  108. data/test/Delete.test.rb +68 -0
  109. data/test/Insert.test.rb +123 -0
  110. data/test/MockPGconn.rb +62 -0
  111. data/test/NullConnection.test.rb +32 -0
  112. data/test/PgBit.test.rb +98 -0
  113. data/test/PgBox.test.rb +108 -0
  114. data/test/PgCidr.test.rb +61 -0
  115. data/test/PgCircle.test.rb +107 -0
  116. data/test/PgInet.test.rb +61 -0
  117. data/test/PgInterval.test.rb +180 -0
  118. data/test/PgLineSegment.test.rb +108 -0
  119. data/test/PgMacAddr.test.rb +61 -0
  120. data/test/PgPath.test.rb +106 -0
  121. data/test/PgPoint.test.rb +100 -0
  122. data/test/PgPolygon.test.rb +95 -0
  123. data/test/PgTime.test.rb +120 -0
  124. data/test/PgTimeWithTimeZone.test.rb +117 -0
  125. data/test/PgTimestamp.test.rb +134 -0
  126. data/test/RandomThings.rb +25 -0
  127. data/test/Savepoint.test.rb +286 -0
  128. data/test/Select.test.rb +930 -0
  129. data/test/Test.rb +62 -0
  130. data/test/TestConfig.rb +21 -0
  131. data/test/TestSetup.rb +13 -0
  132. data/test/TestUtil.rb +92 -0
  133. data/test/Transaction.test.rb +275 -0
  134. data/test/Translate.test.rb +354 -0
  135. data/test/Update.test.rb +227 -0
  136. data/test/roundtrip.test.rb +565 -0
  137. data/test/test +34 -0
  138. data/tools/exampleinserter/ExampleInserter.rb +177 -0
  139. data/tools/rdoc/ChangeLog +796 -0
  140. data/tools/rdoc/EXAMPLE.rb +48 -0
  141. data/tools/rdoc/MANIFEST +58 -0
  142. data/tools/rdoc/Makefile +27 -0
  143. data/tools/rdoc/NEW_FEATURES +226 -0
  144. data/tools/rdoc/README +390 -0
  145. data/tools/rdoc/ToDo +6 -0
  146. data/tools/rdoc/contrib/Index +6 -0
  147. data/tools/rdoc/contrib/xslfo/ChangeLog +181 -0
  148. data/tools/rdoc/contrib/xslfo/README +106 -0
  149. data/tools/rdoc/contrib/xslfo/TODO +10 -0
  150. data/tools/rdoc/contrib/xslfo/convert.xsl +151 -0
  151. data/tools/rdoc/contrib/xslfo/demo/README +21 -0
  152. data/tools/rdoc/contrib/xslfo/demo/rdocfo +99 -0
  153. data/tools/rdoc/contrib/xslfo/fcm.xsl +54 -0
  154. data/tools/rdoc/contrib/xslfo/files.xsl +62 -0
  155. data/tools/rdoc/contrib/xslfo/labeled-lists.xsl +66 -0
  156. data/tools/rdoc/contrib/xslfo/lists.xsl +44 -0
  157. data/tools/rdoc/contrib/xslfo/modules.xsl +152 -0
  158. data/tools/rdoc/contrib/xslfo/rdoc.xsl +75 -0
  159. data/tools/rdoc/contrib/xslfo/source.xsl +66 -0
  160. data/tools/rdoc/contrib/xslfo/styles.xsl +69 -0
  161. data/tools/rdoc/contrib/xslfo/tables.xsl +67 -0
  162. data/tools/rdoc/contrib/xslfo/utils.xsl +21 -0
  163. data/tools/rdoc/debian/changelog +33 -0
  164. data/tools/rdoc/debian/compat +1 -0
  165. data/tools/rdoc/debian/control +20 -0
  166. data/tools/rdoc/debian/copyright +10 -0
  167. data/tools/rdoc/debian/dirs +2 -0
  168. data/tools/rdoc/debian/docs +2 -0
  169. data/tools/rdoc/debian/rdoc.1 +252 -0
  170. data/tools/rdoc/debian/rdoc.manpages +1 -0
  171. data/tools/rdoc/debian/rdoc.pod +149 -0
  172. data/tools/rdoc/debian/rules +9 -0
  173. data/tools/rdoc/dot/dot.rb +255 -0
  174. data/tools/rdoc/etc/rdoc.dtd +203 -0
  175. data/tools/rdoc/install.rb +137 -0
  176. data/tools/rdoc/markup/install.rb +43 -0
  177. data/tools/rdoc/markup/sample/sample.rb +42 -0
  178. data/tools/rdoc/markup/simple_markup/fragments.rb +323 -0
  179. data/tools/rdoc/markup/simple_markup/inline.rb +348 -0
  180. data/tools/rdoc/markup/simple_markup/lines.rb +147 -0
  181. data/tools/rdoc/markup/simple_markup/preprocess.rb +68 -0
  182. data/tools/rdoc/markup/simple_markup/to_html.rb +281 -0
  183. data/tools/rdoc/markup/simple_markup.rb +474 -0
  184. data/tools/rdoc/markup/test/AllTests.rb +2 -0
  185. data/tools/rdoc/markup/test/TestInline.rb +151 -0
  186. data/tools/rdoc/markup/test/TestParse.rb +411 -0
  187. data/tools/rdoc/rdoc/code_objects.rb +536 -0
  188. data/tools/rdoc/rdoc/diagram.rb +331 -0
  189. data/tools/rdoc/rdoc/generators/chm_generator.rb +112 -0
  190. data/tools/rdoc/rdoc/generators/html_generator.rb +1268 -0
  191. data/tools/rdoc/rdoc/generators/template/chm/chm.rb +86 -0
  192. data/tools/rdoc/rdoc/generators/template/html/html.rb +705 -0
  193. data/tools/rdoc/rdoc/generators/template/html/kilmer.rb +377 -0
  194. data/tools/rdoc/rdoc/generators/template/xml/rdf.rb +110 -0
  195. data/tools/rdoc/rdoc/generators/template/xml/xml.rb +110 -0
  196. data/tools/rdoc/rdoc/generators/xml_generator.rb +130 -0
  197. data/tools/rdoc/rdoc/options.rb +451 -0
  198. data/tools/rdoc/rdoc/parsers/parse_c.rb +287 -0
  199. data/tools/rdoc/rdoc/parsers/parse_f95.rb +118 -0
  200. data/tools/rdoc/rdoc/parsers/parse_rb.rb +2311 -0
  201. data/tools/rdoc/rdoc/parsers/parse_simple.rb +37 -0
  202. data/tools/rdoc/rdoc/parsers/parserfactory.rb +75 -0
  203. data/tools/rdoc/rdoc/rdoc.rb +219 -0
  204. data/tools/rdoc/rdoc/template.rb +234 -0
  205. data/tools/rdoc/rdoc/tokenstream.rb +25 -0
  206. data/tools/rdoc/rdoc.rb +9 -0
  207. metadata +291 -0
@@ -0,0 +1,2311 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Parse a Ruby source file, building a set of objects
4
+ # representing the modules, classes, methods,
5
+ # requires, and includes we find (these classes
6
+ # are defined in code_objects.rb).
7
+
8
+ # This file contains stuff stolen outright from:
9
+ #
10
+ # rtags.rb -
11
+ # ruby-lex.rb - ruby lexcal analizer
12
+ # ruby-token.rb - ruby tokens
13
+ # by Keiju ISHITSUKA (Nippon Rational Inc.)
14
+ #
15
+
16
+ require "tracer"
17
+ require "e2mmap"
18
+ require "irb/slex"
19
+
20
+ require "rdoc/code_objects"
21
+ require "rdoc/tokenstream"
22
+
23
+ require "markup/simple_markup/preprocess"
24
+
25
+ require "rdoc/parsers/parserfactory"
26
+
27
+ $TOKEN_DEBUG = $DEBUG
28
+
29
+ # Definitions of all tokens involved in the lexical analysis
30
+
31
+ module RubyToken
32
+ EXPR_BEG = :EXPR_BEG
33
+ EXPR_MID = :EXPR_MID
34
+ EXPR_END = :EXPR_END
35
+ EXPR_ARG = :EXPR_ARG
36
+ EXPR_FNAME = :EXPR_FNAME
37
+ EXPR_DOT = :EXPR_DOT
38
+ EXPR_CLASS = :EXPR_CLASS
39
+
40
+ class Token
41
+ NO_TEXT = "??".freeze
42
+ attr :text
43
+
44
+ def initialize(line_no, char_no)
45
+ @line_no = line_no
46
+ @char_no = char_no
47
+ @text = NO_TEXT
48
+ end
49
+
50
+ # Because we're used in contexts that expect to return a token,
51
+ # we set the text string and then return ourselves
52
+ def set_text(text)
53
+ @text = text
54
+ self
55
+ end
56
+
57
+ attr_reader :line_no, :char_no, :text
58
+ end
59
+
60
+ class TkNode < Token
61
+ attr :node
62
+ end
63
+
64
+ class TkId < Token
65
+ def initialize(line_no, char_no, name)
66
+ super(line_no, char_no)
67
+ @name = name
68
+ end
69
+ attr :name
70
+ end
71
+
72
+ class TkKW < TkId
73
+ end
74
+
75
+ class TkVal < Token
76
+ def initialize(line_no, char_no, value = nil)
77
+ super(line_no, char_no)
78
+ set_text(value)
79
+ end
80
+ end
81
+
82
+ class TkOp < Token
83
+ def name
84
+ self.class.op_name
85
+ end
86
+ end
87
+
88
+ class TkOPASGN < TkOp
89
+ def initialize(line_no, char_no, op)
90
+ super(line_no, char_no)
91
+ op = TkReading2Token[op] unless op.kind_of?(Symbol)
92
+ @op = op
93
+ end
94
+ attr :op
95
+ end
96
+
97
+ class TkUnknownChar < Token
98
+ def initialize(line_no, char_no, id)
99
+ super(line_no, char_no)
100
+ @name = char_no.chr
101
+ end
102
+ attr :name
103
+ end
104
+
105
+ class TkError < Token
106
+ end
107
+
108
+ def set_token_position(line, char)
109
+ @prev_line_no = line
110
+ @prev_char_no = char
111
+ end
112
+
113
+ def Token(token, value = nil)
114
+ tk = nil
115
+ case token
116
+ when String, Symbol
117
+ source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
118
+ if (tk = source[token]).nil?
119
+ IRB.fail TkReading2TokenNoKey, token
120
+ end
121
+ tk = Token(tk[0], value)
122
+ else
123
+ tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
124
+ token.new(@prev_line_no, @prev_char_no)
125
+ else
126
+ token.new(@prev_line_no, @prev_char_no, value)
127
+ end
128
+ end
129
+ tk
130
+ end
131
+
132
+ TokenDefinitions = [
133
+ [:TkCLASS, TkKW, "class", EXPR_CLASS],
134
+ [:TkMODULE, TkKW, "module", EXPR_BEG],
135
+ [:TkDEF, TkKW, "def", EXPR_FNAME],
136
+ [:TkUNDEF, TkKW, "undef", EXPR_FNAME],
137
+ [:TkBEGIN, TkKW, "begin", EXPR_BEG],
138
+ [:TkRESCUE, TkKW, "rescue", EXPR_MID],
139
+ [:TkENSURE, TkKW, "ensure", EXPR_BEG],
140
+ [:TkEND, TkKW, "end", EXPR_END],
141
+ [:TkIF, TkKW, "if", EXPR_BEG, :TkIF_MOD],
142
+ [:TkUNLESS, TkKW, "unless", EXPR_BEG, :TkUNLESS_MOD],
143
+ [:TkTHEN, TkKW, "then", EXPR_BEG],
144
+ [:TkELSIF, TkKW, "elsif", EXPR_BEG],
145
+ [:TkELSE, TkKW, "else", EXPR_BEG],
146
+ [:TkCASE, TkKW, "case", EXPR_BEG],
147
+ [:TkWHEN, TkKW, "when", EXPR_BEG],
148
+ [:TkWHILE, TkKW, "while", EXPR_BEG, :TkWHILE_MOD],
149
+ [:TkUNTIL, TkKW, "until", EXPR_BEG, :TkUNTIL_MOD],
150
+ [:TkFOR, TkKW, "for", EXPR_BEG],
151
+ [:TkBREAK, TkKW, "break", EXPR_END],
152
+ [:TkNEXT, TkKW, "next", EXPR_END],
153
+ [:TkREDO, TkKW, "redo", EXPR_END],
154
+ [:TkRETRY, TkKW, "retry", EXPR_END],
155
+ [:TkIN, TkKW, "in", EXPR_BEG],
156
+ [:TkDO, TkKW, "do", EXPR_BEG],
157
+ [:TkRETURN, TkKW, "return", EXPR_MID],
158
+ [:TkYIELD, TkKW, "yield", EXPR_END],
159
+ [:TkSUPER, TkKW, "super", EXPR_END],
160
+ [:TkSELF, TkKW, "self", EXPR_END],
161
+ [:TkNIL, TkKW, "nil", EXPR_END],
162
+ [:TkTRUE, TkKW, "true", EXPR_END],
163
+ [:TkFALSE, TkKW, "false", EXPR_END],
164
+ [:TkAND, TkKW, "and", EXPR_BEG],
165
+ [:TkOR, TkKW, "or", EXPR_BEG],
166
+ [:TkNOT, TkKW, "not", EXPR_BEG],
167
+ [:TkIF_MOD, TkKW],
168
+ [:TkUNLESS_MOD, TkKW],
169
+ [:TkWHILE_MOD, TkKW],
170
+ [:TkUNTIL_MOD, TkKW],
171
+ [:TkALIAS, TkKW, "alias", EXPR_FNAME],
172
+ [:TkDEFINED, TkKW, "defined?", EXPR_END],
173
+ [:TklBEGIN, TkKW, "BEGIN", EXPR_END],
174
+ [:TklEND, TkKW, "END", EXPR_END],
175
+ [:Tk__LINE__, TkKW, "__LINE__", EXPR_END],
176
+ [:Tk__FILE__, TkKW, "__FILE__", EXPR_END],
177
+
178
+ [:TkIDENTIFIER, TkId],
179
+ [:TkFID, TkId],
180
+ [:TkGVAR, TkId],
181
+ [:TkIVAR, TkId],
182
+ [:TkCONSTANT, TkId],
183
+
184
+ [:TkINTEGER, TkVal],
185
+ [:TkFLOAT, TkVal],
186
+ [:TkSTRING, TkVal],
187
+ [:TkXSTRING, TkVal],
188
+ [:TkREGEXP, TkVal],
189
+ [:TkCOMMENT, TkVal],
190
+
191
+ [:TkDSTRING, TkNode],
192
+ [:TkDXSTRING, TkNode],
193
+ [:TkDREGEXP, TkNode],
194
+ [:TkNTH_REF, TkId],
195
+ [:TkBACK_REF, TkId],
196
+
197
+ [:TkUPLUS, TkOp, "+@"],
198
+ [:TkUMINUS, TkOp, "-@"],
199
+ [:TkPOW, TkOp, "**"],
200
+ [:TkCMP, TkOp, "<=>"],
201
+ [:TkEQ, TkOp, "=="],
202
+ [:TkEQQ, TkOp, "==="],
203
+ [:TkNEQ, TkOp, "!="],
204
+ [:TkGEQ, TkOp, ">="],
205
+ [:TkLEQ, TkOp, "<="],
206
+ [:TkANDOP, TkOp, "&&"],
207
+ [:TkOROP, TkOp, "||"],
208
+ [:TkMATCH, TkOp, "=~"],
209
+ [:TkNMATCH, TkOp, "!~"],
210
+ [:TkDOT2, TkOp, ".."],
211
+ [:TkDOT3, TkOp, "..."],
212
+ [:TkAREF, TkOp, "[]"],
213
+ [:TkASET, TkOp, "[]="],
214
+ [:TkLSHFT, TkOp, "<<"],
215
+ [:TkRSHFT, TkOp, ">>"],
216
+ [:TkCOLON2, TkOp],
217
+ [:TkCOLON3, TkOp],
218
+ # [:OPASGN, TkOp], # +=, -= etc. #
219
+ [:TkASSOC, TkOp, "=>"],
220
+ [:TkQUESTION, TkOp, "?"], #?
221
+ [:TkCOLON, TkOp, ":"], #:
222
+
223
+ [:TkfLPAREN], # func( #
224
+ [:TkfLBRACK], # func[ #
225
+ [:TkfLBRACE], # func{ #
226
+ [:TkSTAR], # *arg
227
+ [:TkAMPER], # &arg #
228
+ [:TkSYMBOL, TkId], # :SYMBOL
229
+ [:TkSYMBEG, TkId],
230
+ [:TkGT, TkOp, ">"],
231
+ [:TkLT, TkOp, "<"],
232
+ [:TkPLUS, TkOp, "+"],
233
+ [:TkMINUS, TkOp, "-"],
234
+ [:TkMULT, TkOp, "*"],
235
+ [:TkDIV, TkOp, "/"],
236
+ [:TkMOD, TkOp, "%"],
237
+ [:TkBITOR, TkOp, "|"],
238
+ [:TkBITXOR, TkOp, "^"],
239
+ [:TkBITAND, TkOp, "&"],
240
+ [:TkBITNOT, TkOp, "~"],
241
+ [:TkNOTOP, TkOp, "!"],
242
+
243
+ [:TkBACKQUOTE, TkOp, "`"],
244
+
245
+ [:TkASSGIN, Token, "="],
246
+ [:TkDOT, Token, "."],
247
+ [:TkLPAREN, Token, "("], #(exp)
248
+ [:TkLBRACK, Token, "["], #[arry]
249
+ [:TkLBRACE, Token, "{"], #{hash}
250
+ [:TkRPAREN, Token, ")"],
251
+ [:TkRBRACK, Token, "]"],
252
+ [:TkRBRACE, Token, "}"],
253
+ [:TkCOMMA, Token, ","],
254
+ [:TkSEMICOLON, Token, ";"],
255
+
256
+ [:TkRD_COMMENT],
257
+ [:TkSPACE],
258
+ [:TkNL],
259
+ [:TkEND_OF_SCRIPT],
260
+
261
+ [:TkBACKSLASH, TkUnknownChar, "\\"],
262
+ [:TkAT, TkUnknownChar, "@"],
263
+ [:TkDOLLAR, TkUnknownChar, "\$"], #"
264
+ ]
265
+
266
+ # {reading => token_class}
267
+ # {reading => [token_class, *opt]}
268
+ TkReading2Token = {}
269
+ TkSymbol2Token = {}
270
+
271
+ def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
272
+ token_n = token_n.id2name unless token_n.kind_of?(String)
273
+ if RubyToken.const_defined?(token_n)
274
+ IRB.fail AlreadyDefinedToken, token_n
275
+ end
276
+
277
+ token_c = Class.new super_token
278
+ RubyToken.const_set token_n, token_c
279
+ # token_c.inspect
280
+
281
+ if reading
282
+ if TkReading2Token[reading]
283
+ IRB.fail TkReading2TokenDuplicateError, token_n, reading
284
+ end
285
+ if opts.empty?
286
+ TkReading2Token[reading] = [token_c]
287
+ else
288
+ TkReading2Token[reading] = [token_c].concat(opts)
289
+ end
290
+ end
291
+ TkSymbol2Token[token_n.intern] = token_c
292
+
293
+ if token_c <= TkOp
294
+ token_c.class_eval %{
295
+ def self.op_name; "#{reading}"; end
296
+ }
297
+ end
298
+ end
299
+
300
+ for defs in TokenDefinitions
301
+ def_token(*defs)
302
+ end
303
+
304
+ NEWLINE_TOKEN = TkNL.new(0,0)
305
+ NEWLINE_TOKEN.set_text("\n")
306
+
307
+ end
308
+
309
+
310
+
311
+ # Lexical analyzer for Ruby source
312
+
313
+ class RubyLex
314
+
315
+ ######################################################################
316
+ #
317
+ # Read an input stream character by character. We allow for unlimited
318
+ # ungetting of characters just read.
319
+ #
320
+ # We simplify the implementation greatly by reading the entire input
321
+ # into a buffer initially, and then simply traversing it using
322
+ # pointers.
323
+ #
324
+ # We also have to allow for the <i>here document diversion</i>. This
325
+ # little gem comes about when the lexer encounters a here
326
+ # document. At this point we effectively need to split the input
327
+ # stream into two parts: one to read the body of the here document,
328
+ # the other to read the rest of the input line where the here
329
+ # document was initially encountered. For example, we might have
330
+ #
331
+ # do_something(<<-A, <<-B)
332
+ # stuff
333
+ # for
334
+ # A
335
+ # stuff
336
+ # for
337
+ # B
338
+ #
339
+ # When the lexer encounters the <<A, it reads until the end of the
340
+ # line, and keeps it around for later. It then reads the body of the
341
+ # here document. Once complete, it needs to read the rest of the
342
+ # original line, but then skip the here document body.
343
+ #
344
+
345
+ class BufferedReader
346
+
347
+ attr_reader :line_num
348
+
349
+ def initialize(content)
350
+ if /\t/ =~ content
351
+ tab_width = Options.instance.tab_width
352
+ content = content.split(/\n/).map do |line|
353
+ 1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)} && $~ #`
354
+ line
355
+ end .join("\n")
356
+ end
357
+ @content = content
358
+ @content << "\n" unless @content[-1,1] == "\n"
359
+ @size = @content.size
360
+ @offset = 0
361
+ @hwm = 0
362
+ @line_num = 1
363
+ @read_back_offset = 0
364
+ @last_newline = 0
365
+ @newline_pending = false
366
+ end
367
+
368
+ def column
369
+ @offset - @last_newline
370
+ end
371
+
372
+ def getc
373
+ return nil if @offset >= @size
374
+ ch = @content[@offset, 1]
375
+
376
+ @offset += 1
377
+ @hwm = @offset if @hwm < @offset
378
+
379
+ if @newline_pending
380
+ @line_num += 1
381
+ @last_newline = @offset - 1
382
+ @newline_pending = false
383
+ end
384
+
385
+ if ch == "\n"
386
+ @newline_pending = true
387
+ end
388
+ ch
389
+ end
390
+
391
+ def getc_already_read
392
+ getc
393
+ end
394
+
395
+ def ungetc(ch)
396
+ raise "unget past beginning of file" if @offset <= 0
397
+ @offset -= 1
398
+ if @content[@offset] == ?\n
399
+ @newline_pending = false
400
+ end
401
+ end
402
+
403
+ def get_read
404
+ res = @content[@read_back_offset...@offset]
405
+ @read_back_offset = @offset
406
+ res
407
+ end
408
+
409
+ def peek(at)
410
+ pos = @offset + at
411
+ if pos >= @size
412
+ nil
413
+ else
414
+ @content[pos, 1]
415
+ end
416
+ end
417
+
418
+ def peek_equal(str)
419
+ @content[@offset, str.length] == str
420
+ end
421
+
422
+ def divert_read_from(reserve)
423
+ @content[@offset, 0] = reserve
424
+ @size = @content.size
425
+ end
426
+ end
427
+
428
+ # end of nested class BufferedReader
429
+
430
+ extend Exception2MessageMapper
431
+ def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
432
+ def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
433
+ def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
434
+ def_exception(:TkReading2TokenDuplicateError,
435
+ "key duplicate(token_n='%s', key='%s')")
436
+ def_exception(:SyntaxError, "%s")
437
+
438
+ include RubyToken
439
+
440
+ attr_reader :continue
441
+ attr_reader :lex_state
442
+
443
+ def RubyLex.debug?
444
+ false
445
+ end
446
+
447
+ def initialize(content)
448
+ lex_init
449
+
450
+ @reader = BufferedReader.new(content)
451
+
452
+ @exp_line_no = @line_no = 1
453
+ @base_char_no = 0
454
+ @indent = 0
455
+
456
+ @ltype = nil
457
+ @quoted = nil
458
+ @lex_state = EXPR_BEG
459
+ @space_seen = false
460
+
461
+ @continue = false
462
+ @line = ""
463
+
464
+ @skip_space = false
465
+ @read_auto_clean_up = false
466
+ @exception_on_syntax_error = true
467
+ end
468
+
469
+ attr :skip_space, true
470
+ attr :read_auto_clean_up, true
471
+ attr :exception_on_syntax_error, true
472
+
473
+ attr :indent
474
+
475
+ # io functions
476
+ def line_no
477
+ @reader.line_num
478
+ end
479
+
480
+ def char_no
481
+ @reader.column
482
+ end
483
+
484
+ def get_read
485
+ @reader.get_read
486
+ end
487
+
488
+ def getc
489
+ @reader.getc
490
+ end
491
+
492
+ def getc_of_rests
493
+ @reader.getc_already_read
494
+ end
495
+
496
+ def gets
497
+ c = getc or return
498
+ l = ""
499
+ begin
500
+ l.concat c unless c == "\r"
501
+ break if c == "\n"
502
+ end while c = getc
503
+ l
504
+ end
505
+
506
+
507
+ def ungetc(c = nil)
508
+ @reader.ungetc(c)
509
+ end
510
+
511
+ def peek_equal?(str)
512
+ @reader.peek_equal(str)
513
+ end
514
+
515
+ def peek(i = 0)
516
+ @reader.peek(i)
517
+ end
518
+
519
+ def lex
520
+ until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
521
+ !@continue or
522
+ tk.nil?)
523
+ end
524
+ line = get_read
525
+
526
+ if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
527
+ nil
528
+ else
529
+ line
530
+ end
531
+ end
532
+
533
+ def token
534
+ set_token_position(line_no, char_no)
535
+ begin
536
+ begin
537
+ tk = @OP.match(self)
538
+ @space_seen = tk.kind_of?(TkSPACE)
539
+ rescue SyntaxError
540
+ abort if @exception_on_syntax_error
541
+ tk = TkError.new(line_no, char_no)
542
+ end
543
+ end while @skip_space and tk.kind_of?(TkSPACE)
544
+ if @read_auto_clean_up
545
+ get_read
546
+ end
547
+ # throw :eof unless tk
548
+ p tk if $DEBUG
549
+ tk
550
+ end
551
+
552
+ ENINDENT_CLAUSE = [
553
+ "case", "class", "def", "do", "for", "if",
554
+ "module", "unless", "until", "while", "begin" #, "when"
555
+ ]
556
+ DEINDENT_CLAUSE = ["end" #, "when"
557
+ ]
558
+
559
+ PERCENT_LTYPE = {
560
+ "q" => "\'",
561
+ "Q" => "\"",
562
+ "x" => "\`",
563
+ "r" => "\/",
564
+ "w" => "]"
565
+ }
566
+
567
+ PERCENT_PAREN = {
568
+ "{" => "}",
569
+ "[" => "]",
570
+ "<" => ">",
571
+ "(" => ")"
572
+ }
573
+
574
+ Ltype2Token = {
575
+ "\'" => TkSTRING,
576
+ "\"" => TkSTRING,
577
+ "\`" => TkXSTRING,
578
+ "\/" => TkREGEXP,
579
+ "]" => TkDSTRING
580
+ }
581
+ DLtype2Token = {
582
+ "\"" => TkDSTRING,
583
+ "\`" => TkDXSTRING,
584
+ "\/" => TkDREGEXP,
585
+ }
586
+
587
+ def lex_init()
588
+ @OP = SLex.new
589
+ @OP.def_rules("\0", "\004", "\032") do |chars, io|
590
+ Token(TkEND_OF_SCRIPT).set_text(chars)
591
+ end
592
+
593
+ @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io|
594
+ @space_seen = TRUE
595
+ while (ch = getc) =~ /[ \t\f\r\13]/
596
+ chars << ch
597
+ end
598
+ ungetc
599
+ Token(TkSPACE).set_text(chars)
600
+ end
601
+
602
+ @OP.def_rule("#") do
603
+ |op, io|
604
+ identify_comment
605
+ end
606
+
607
+ @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do
608
+ |op, io|
609
+ str = op
610
+ @ltype = "="
611
+
612
+ begin
613
+ ch = getc
614
+ str << ch
615
+ end until ch == "\n"
616
+
617
+ until peek_equal?("=end") && peek(4) =~ /\s/
618
+ begin
619
+ ch = getc
620
+ str << ch
621
+ end until ch == "\n"
622
+ end
623
+
624
+ str << "=end"
625
+ gets
626
+
627
+ @ltype = nil
628
+ Token(TkRD_COMMENT).set_text(str)
629
+ end
630
+
631
+ @OP.def_rule("\n") do
632
+ print "\\n\n" if RubyLex.debug?
633
+ case @lex_state
634
+ when EXPR_BEG, EXPR_FNAME, EXPR_DOT
635
+ @continue = TRUE
636
+ else
637
+ @continue = FALSE
638
+ @lex_state = EXPR_BEG
639
+ end
640
+ Token(TkNL).set_text("\n")
641
+ end
642
+
643
+ @OP.def_rules("*", "**",
644
+ "!", "!=", "!~",
645
+ "=", "==", "===",
646
+ "=~", "<=>",
647
+ "<", "<=",
648
+ ">", ">=", ">>") do
649
+ |op, io|
650
+ @lex_state = EXPR_BEG
651
+ Token(op).set_text(op)
652
+ end
653
+
654
+ @OP.def_rules("<<") do
655
+ |op, io|
656
+ tk = nil
657
+ if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
658
+ (@lex_state != EXPR_ARG || @space_seen)
659
+ c = peek(0)
660
+ if /[-\w_\"\'\`]/ =~ c
661
+ tk = identify_here_document
662
+ end
663
+ end
664
+ if !tk
665
+ @lex_state = EXPR_BEG
666
+ tk = Token(op).set_text(op)
667
+ end
668
+ tk
669
+ end
670
+
671
+ @OP.def_rules("'", '"') do
672
+ |op, io|
673
+ identify_string(op)
674
+ end
675
+
676
+ @OP.def_rules("`") do
677
+ |op, io|
678
+ if @lex_state == EXPR_FNAME
679
+ Token(op).set_text(op)
680
+ else
681
+ identify_string(op)
682
+ end
683
+ end
684
+
685
+ @OP.def_rules('?') do
686
+ |op, io|
687
+ if @lex_state == EXPR_END
688
+ @lex_state = EXPR_BEG
689
+ Token(TkQUESTION).set_text(op)
690
+ else
691
+ ch = getc
692
+ if @lex_state == EXPR_ARG && ch !~ /\s/
693
+ ungetc
694
+ @lex_state = EXPR_BEG;
695
+ Token(TkQUESTION).set_text(op)
696
+ else
697
+ str = op
698
+ str << ch
699
+ if (ch == '\\') #'
700
+ str << read_escape
701
+ end
702
+ @lex_state = EXPR_END
703
+ Token(TkINTEGER).set_text(str)
704
+ end
705
+ end
706
+ end
707
+
708
+ @OP.def_rules("&", "&&", "|", "||") do
709
+ |op, io|
710
+ @lex_state = EXPR_BEG
711
+ Token(op).set_text(op)
712
+ end
713
+
714
+ @OP.def_rules("+=", "-=", "*=", "**=",
715
+ "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
716
+ |op, io|
717
+ @lex_state = EXPR_BEG
718
+ op =~ /^(.*)=$/
719
+ Token(TkOPASGN, $1).set_text(op)
720
+ end
721
+
722
+ @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io|
723
+ Token(TkUPLUS).set_text(op)
724
+ end
725
+
726
+ @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io|
727
+ Token(TkUMINUS).set_text(op)
728
+ end
729
+
730
+ @OP.def_rules("+", "-") do
731
+ |op, io|
732
+ catch(:RET) do
733
+ if @lex_state == EXPR_ARG
734
+ if @space_seen and peek(0) =~ /[0-9]/
735
+ throw :RET, identify_number(op)
736
+ else
737
+ @lex_state = EXPR_BEG
738
+ end
739
+ elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/
740
+ throw :RET, identify_number(op)
741
+ else
742
+ @lex_state = EXPR_BEG
743
+ end
744
+ Token(op).set_text(op)
745
+ end
746
+ end
747
+
748
+ @OP.def_rule(".") do
749
+ @lex_state = EXPR_BEG
750
+ if peek(0) =~ /[0-9]/
751
+ ungetc
752
+ identify_number("")
753
+ else
754
+ # for obj.if
755
+ @lex_state = EXPR_DOT
756
+ Token(TkDOT).set_text(".")
757
+ end
758
+ end
759
+
760
+ @OP.def_rules("..", "...") do
761
+ |op, io|
762
+ @lex_state = EXPR_BEG
763
+ Token(op).set_text(op)
764
+ end
765
+
766
+ lex_int2
767
+ end
768
+
769
+ def lex_int2
770
+ @OP.def_rules("]", "}", ")") do
771
+ |op, io|
772
+ @lex_state = EXPR_END
773
+ @indent -= 1
774
+ Token(op).set_text(op)
775
+ end
776
+
777
+ @OP.def_rule(":") do
778
+ if @lex_state == EXPR_END || peek(0) =~ /\s/
779
+ @lex_state = EXPR_BEG
780
+ tk = Token(TkCOLON)
781
+ else
782
+ @lex_state = EXPR_FNAME;
783
+ tk = Token(TkSYMBEG)
784
+ end
785
+ tk.set_text(":")
786
+ end
787
+
788
+ @OP.def_rule("::") do
789
+ # p @lex_state.id2name, @space_seen
790
+ if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen
791
+ @lex_state = EXPR_BEG
792
+ tk = Token(TkCOLON3)
793
+ else
794
+ @lex_state = EXPR_DOT
795
+ tk = Token(TkCOLON2)
796
+ end
797
+ tk.set_text("::")
798
+ end
799
+
800
+ @OP.def_rule("/") do
801
+ |op, io|
802
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
803
+ identify_string(op)
804
+ elsif peek(0) == '='
805
+ getc
806
+ @lex_state = EXPR_BEG
807
+ Token(TkOPASGN, :/).set_text("/=") #")
808
+ elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
809
+ identify_string(op)
810
+ else
811
+ @lex_state = EXPR_BEG
812
+ Token("/").set_text(op)
813
+ end
814
+ end
815
+
816
+ @OP.def_rules("^") do
817
+ @lex_state = EXPR_BEG
818
+ Token("^").set_text("^")
819
+ end
820
+
821
+ # @OP.def_rules("^=") do
822
+ # @lex_state = EXPR_BEG
823
+ # Token(TkOPASGN, :^)
824
+ # end
825
+
826
+ @OP.def_rules(",", ";") do
827
+ |op, io|
828
+ @lex_state = EXPR_BEG
829
+ Token(op).set_text(op)
830
+ end
831
+
832
+ @OP.def_rule("~") do
833
+ @lex_state = EXPR_BEG
834
+ Token("~").set_text("~")
835
+ end
836
+
837
+ @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
838
+ @lex_state = EXPR_BEG
839
+ Token("~").set_text("~@")
840
+ end
841
+
842
+ @OP.def_rule("(") do
843
+ @indent += 1
844
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
845
+ @lex_state = EXPR_BEG
846
+ tk = Token(TkfLPAREN)
847
+ else
848
+ @lex_state = EXPR_BEG
849
+ tk = Token(TkLPAREN)
850
+ end
851
+ tk.set_text("(")
852
+ end
853
+
854
+ @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do
855
+ Token("[]").set_text("[]")
856
+ end
857
+
858
+ @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do
859
+ Token("[]=").set_text("[]=")
860
+ end
861
+
862
+ @OP.def_rule("[") do
863
+ @indent += 1
864
+ if @lex_state == EXPR_FNAME
865
+ t = Token(TkfLBRACK)
866
+ else
867
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
868
+ t = Token(TkLBRACK)
869
+ elsif @lex_state == EXPR_ARG && @space_seen
870
+ t = Token(TkLBRACK)
871
+ else
872
+ t = Token(TkfLBRACK)
873
+ end
874
+ @lex_state = EXPR_BEG
875
+ end
876
+ t.set_text("[")
877
+ end
878
+
879
+ @OP.def_rule("{") do
880
+ @indent += 1
881
+ if @lex_state != EXPR_END && @lex_state != EXPR_ARG
882
+ t = Token(TkLBRACE)
883
+ else
884
+ t = Token(TkfLBRACE)
885
+ end
886
+ @lex_state = EXPR_BEG
887
+ t.set_text("{")
888
+ end
889
+
890
+ @OP.def_rule('\\') do #'
891
+ if getc == "\n"
892
+ @space_seen = true
893
+ @continue = true
894
+ Token(TkSPACE).set_text("\\\n")
895
+ else
896
+ ungetc
897
+ Token("\\").set_text("\\") #"
898
+ end
899
+ end
900
+
901
+ @OP.def_rule('%') do
902
+ |op, io|
903
+ if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
904
+ identify_quotation('%')
905
+ elsif peek(0) == '='
906
+ getc
907
+ Token(TkOPASGN, "%").set_text("%=")
908
+ elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
909
+ identify_quotation('%')
910
+ else
911
+ @lex_state = EXPR_BEG
912
+ Token("%").set_text("%")
913
+ end
914
+ end
915
+
916
+ @OP.def_rule('$') do #'
917
+ identify_gvar
918
+ end
919
+
920
+ @OP.def_rule('@') do
921
+ if peek(0) =~ /[\w_]/
922
+ ungetc
923
+ identify_identifier
924
+ else
925
+ Token("@").set_text("@")
926
+ end
927
+ end
928
+
929
+ # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
930
+ # |op, io|
931
+ # @indent += 1
932
+ # @lex_state = EXPR_FNAME
933
+ # # @lex_state = EXPR_END
934
+ # # until @rests[0] == "\n" or @rests[0] == ";"
935
+ # # rests.shift
936
+ # # end
937
+ # end
938
+
939
+ @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do
940
+ throw :eof
941
+ end
942
+
943
+ @OP.def_rule("") do
944
+ |op, io|
945
+ printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug?
946
+ if peek(0) =~ /[0-9]/
947
+ t = identify_number("")
948
+ elsif peek(0) =~ /[\w_]/
949
+ t = identify_identifier
950
+ end
951
+ printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
952
+ t
953
+ end
954
+
955
+ p @OP if RubyLex.debug?
956
+ end
957
+
958
+ def identify_gvar
959
+ @lex_state = EXPR_END
960
+ str = "$"
961
+
962
+ tk = case ch = getc
963
+ when /[~_*$?!@\/\\;,=:<>".]/ #"
964
+ str << ch
965
+ Token(TkGVAR, str)
966
+
967
+ when "-"
968
+ str << "-" << getc
969
+ Token(TkGVAR, str)
970
+
971
+ when "&", "`", "'", "+"
972
+ str << ch
973
+ Token(TkBACK_REF, str)
974
+
975
+ when /[1-9]/
976
+ str << ch
977
+ while (ch = getc) =~ /[0-9]/
978
+ str << ch
979
+ end
980
+ ungetc
981
+ Token(TkNTH_REF)
982
+ when /\w/
983
+ ungetc
984
+ ungetc
985
+ return identify_identifier
986
+ else
987
+ ungetc
988
+ Token("$")
989
+ end
990
+ tk.set_text(str)
991
+ end
992
+
993
+ def identify_identifier
994
+ token = ""
995
+ token.concat getc if peek(0) =~ /[$@]/
996
+ while (ch = getc) =~ /\w|_/
997
+ print ":", ch, ":" if RubyLex.debug?
998
+ token.concat ch
999
+ end
1000
+ ungetc
1001
+
1002
+ if ch == "!" or ch == "?"
1003
+ token.concat getc
1004
+ end
1005
+ # fix token
1006
+
1007
+ # puts "identifier - #{token}, state = #@lex_state"
1008
+
1009
+ case token
1010
+ when /^\$/
1011
+ return Token(TkGVAR, token).set_text(token)
1012
+ when /^\@/
1013
+ @lex_state = EXPR_END
1014
+ return Token(TkIVAR, token).set_text(token)
1015
+ end
1016
+
1017
+ if @lex_state != EXPR_DOT
1018
+ print token, "\n" if RubyLex.debug?
1019
+
1020
+ token_c, *trans = TkReading2Token[token]
1021
+ if token_c
1022
+ # reserved word?
1023
+
1024
+ if (@lex_state != EXPR_BEG &&
1025
+ @lex_state != EXPR_FNAME &&
1026
+ trans[1])
1027
+ # modifiers
1028
+ token_c = TkSymbol2Token[trans[1]]
1029
+ @lex_state = trans[0]
1030
+ else
1031
+ if @lex_state != EXPR_FNAME
1032
+ if ENINDENT_CLAUSE.include?(token)
1033
+ @indent += 1
1034
+ elsif DEINDENT_CLAUSE.include?(token)
1035
+ @indent -= 1
1036
+ end
1037
+ @lex_state = trans[0]
1038
+ else
1039
+ @lex_state = EXPR_END
1040
+ end
1041
+ end
1042
+ return Token(token_c, token).set_text(token)
1043
+ end
1044
+ end
1045
+
1046
+ if @lex_state == EXPR_FNAME
1047
+ @lex_state = EXPR_END
1048
+ if peek(0) == '='
1049
+ token.concat getc
1050
+ end
1051
+ elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT
1052
+ @lex_state = EXPR_ARG
1053
+ else
1054
+ @lex_state = EXPR_END
1055
+ end
1056
+
1057
+ if token[0, 1] =~ /[A-Z]/
1058
+ return Token(TkCONSTANT, token).set_text(token)
1059
+ elsif token[token.size - 1, 1] =~ /[!?]/
1060
+ return Token(TkFID, token).set_text(token)
1061
+ else
1062
+ return Token(TkIDENTIFIER, token).set_text(token)
1063
+ end
1064
+ end
1065
+
1066
+ def identify_here_document
1067
+ ch = getc
1068
+ if ch == "-"
1069
+ ch = getc
1070
+ indent = true
1071
+ end
1072
+ if /['"`]/ =~ ch # '
1073
+ lt = ch
1074
+ quoted = ""
1075
+ while (c = getc) && c != lt
1076
+ quoted.concat c
1077
+ end
1078
+ else
1079
+ lt = '"'
1080
+ quoted = ch.dup
1081
+ while (c = getc) && c =~ /\w/
1082
+ quoted.concat c
1083
+ end
1084
+ ungetc
1085
+ end
1086
+
1087
+ ltback, @ltype = @ltype, lt
1088
+ reserve = ""
1089
+
1090
+ while ch = getc
1091
+ reserve << ch
1092
+ if ch == "\\" #"
1093
+ ch = getc
1094
+ reserve << ch
1095
+ elsif ch == "\n"
1096
+ break
1097
+ end
1098
+ end
1099
+
1100
+ str = ""
1101
+ while (l = gets) && (indent ? l.strip : l) != quoted
1102
+ str << l.chomp << "\n"
1103
+ end
1104
+
1105
+ @reader.divert_read_from(reserve)
1106
+
1107
+ @ltype = ltback
1108
+ @lex_state = EXPR_END
1109
+ Token(Ltype2Token[lt], str).set_text(str.dump)
1110
+ end
1111
+
1112
+ def identify_quotation(initial_char)
1113
+ ch = getc
1114
+ if lt = PERCENT_LTYPE[ch]
1115
+ ch = getc
1116
+ elsif ch =~ /\W/
1117
+ lt = "\""
1118
+ else
1119
+ RubyLex.fail SyntaxError, "unknown type of %string ('#{ch}')"
1120
+ end
1121
+ # if ch !~ /\W/
1122
+ # ungetc
1123
+ # next
1124
+ # end
1125
+ #@ltype = lt
1126
+ @quoted = ch unless @quoted = PERCENT_PAREN[ch]
1127
+ identify_string(lt, @quoted, ch, initial_char)
1128
+ end
1129
+
1130
+ def identify_number(start)
1131
+ str = start.dup
1132
+
1133
+ if start == "+" or start == "-" or start == ""
1134
+ start = getc
1135
+ str << start
1136
+ end
1137
+
1138
+ @lex_state = EXPR_END
1139
+
1140
+ if start == "0"
1141
+ if peek(0) == "x"
1142
+ ch = getc
1143
+ str << ch
1144
+ match = /[0-9a-f_]/
1145
+ else
1146
+ match = /[0-7_]/
1147
+ end
1148
+ while ch = getc
1149
+ if ch !~ match
1150
+ ungetc
1151
+ break
1152
+ else
1153
+ str << ch
1154
+ end
1155
+ end
1156
+ return Token(TkINTEGER).set_text(str)
1157
+ end
1158
+
1159
+ type = TkINTEGER
1160
+ allow_point = TRUE
1161
+ allow_e = TRUE
1162
+ while ch = getc
1163
+ case ch
1164
+ when /[0-9_]/
1165
+ str << ch
1166
+
1167
+ when allow_point && "."
1168
+ type = TkFLOAT
1169
+ if peek(0) !~ /[0-9]/
1170
+ ungetc
1171
+ break
1172
+ end
1173
+ str << ch
1174
+ allow_point = false
1175
+
1176
+ when allow_e && "e", allow_e && "E"
1177
+ str << ch
1178
+ type = TkFLOAT
1179
+ if peek(0) =~ /[+-]/
1180
+ str << getc
1181
+ end
1182
+ allow_e = false
1183
+ allow_point = false
1184
+ else
1185
+ ungetc
1186
+ break
1187
+ end
1188
+ end
1189
+ Token(type).set_text(str)
1190
+ end
1191
+
1192
+ def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
1193
+ @ltype = ltype
1194
+ @quoted = quoted
1195
+ subtype = nil
1196
+
1197
+ str = ""
1198
+ str << initial_char if initial_char
1199
+ str << (opener||quoted)
1200
+
1201
+ nest = 0
1202
+ begin
1203
+ while ch = getc
1204
+ str << ch
1205
+ if @quoted == ch
1206
+ if nest == 0
1207
+ break
1208
+ else
1209
+ nest -= 1
1210
+ end
1211
+ elsif opener == ch
1212
+ nest += 1
1213
+ elsif @ltype != "'" && @ltype != "]" and ch == "#"
1214
+ subtype = true
1215
+ elsif ch == '\\' #'
1216
+ str << read_escape
1217
+ end
1218
+ end
1219
+ if @ltype == "/"
1220
+ if peek(0) =~ /i|o|n|e|s/
1221
+ str << getc
1222
+ end
1223
+ end
1224
+ if subtype
1225
+ Token(DLtype2Token[ltype], str)
1226
+ else
1227
+ Token(Ltype2Token[ltype], str)
1228
+ end.set_text(str)
1229
+ ensure
1230
+ @ltype = nil
1231
+ @quoted = nil
1232
+ @lex_state = EXPR_END
1233
+ end
1234
+ end
1235
+
1236
+ def identify_comment
1237
+ @ltype = "#"
1238
+ comment = "#"
1239
+ while ch = getc
1240
+ if ch == "\\"
1241
+ ch = getc
1242
+ if ch == "\n"
1243
+ ch = " "
1244
+ else
1245
+ comment << "\\"
1246
+ end
1247
+ else
1248
+ if ch == "\n"
1249
+ @ltype = nil
1250
+ ungetc
1251
+ break
1252
+ end
1253
+ end
1254
+ comment << ch
1255
+ end
1256
+ return Token(TkCOMMENT).set_text(comment)
1257
+ end
1258
+
1259
+ def read_escape
1260
+ res = ""
1261
+ case ch = getc
1262
+ when /[0-7]/
1263
+ ungetc ch
1264
+ 3.times do
1265
+ case ch = getc
1266
+ when /[0-7]/
1267
+ when nil
1268
+ break
1269
+ else
1270
+ ungetc
1271
+ break
1272
+ end
1273
+ res << ch
1274
+ end
1275
+
1276
+ when "x"
1277
+ res << ch
1278
+ 2.times do
1279
+ case ch = getc
1280
+ when /[0-9a-fA-F]/
1281
+ when nil
1282
+ break
1283
+ else
1284
+ ungetc
1285
+ break
1286
+ end
1287
+ res << ch
1288
+ end
1289
+
1290
+ when "M"
1291
+ res << ch
1292
+ if (ch = getc) != '-'
1293
+ ungetc
1294
+ else
1295
+ res << ch
1296
+ if (ch = getc) == "\\" #"
1297
+ res << ch
1298
+ res << read_escape
1299
+ else
1300
+ res << ch
1301
+ end
1302
+ end
1303
+
1304
+ when "C", "c", "^"
1305
+ res << ch
1306
+ if ch == "C" and (ch = getc) != "-"
1307
+ ungetc
1308
+ else
1309
+ res << ch
1310
+ if (ch = getc) == "\\" #"
1311
+ res << ch
1312
+ res << read_escape
1313
+ else
1314
+ res << ch
1315
+ end
1316
+ end
1317
+ else
1318
+ res << ch
1319
+ end
1320
+ res
1321
+ end
1322
+ end
1323
+
1324
+
1325
+
1326
+ # Extract code elements from a source file, returning a TopLevel
1327
+ # object containing the constituent file elements.
1328
+ #
1329
+ # This file is based on rtags
1330
+
1331
+ module RDoc
1332
+
1333
+ CLASS_MODIFIERS = [ 'nodoc' ]
1334
+
1335
+ ATTR_MODIFIERS = [ 'nodoc' ]
1336
+
1337
+ METHOD_MODIFIERS = CLASS_MODIFIERS +
1338
+ [ 'arg', 'args', 'yield', 'yields', 'notnew', 'not-new', 'not_new', 'doc' ]
1339
+
1340
+
1341
+ class RubyParser
1342
+ include RubyToken
1343
+ include TokenStream
1344
+
1345
+ extend ParserFactory
1346
+
1347
+ parse_files_matching(/\.rbw?$/)
1348
+
1349
+
1350
+ def initialize(top_level, file_name, content, options)
1351
+ @options = options
1352
+ @size = 0
1353
+ @token_listeners = nil
1354
+ @input_file_name = file_name
1355
+ @scanner = RubyLex.new(content)
1356
+ @scanner.exception_on_syntax_error = false
1357
+ @top_level = top_level
1358
+ @progress = $stderr unless options.quiet
1359
+ end
1360
+
1361
+ def scan
1362
+ @tokens = []
1363
+ @unget_read = []
1364
+ @read = []
1365
+ catch(:eof) do
1366
+ parse_statements(@top_level)
1367
+ end
1368
+ @top_level
1369
+ end
1370
+
1371
+ private
1372
+
1373
+ def warn(msg)
1374
+ prefix = "\n" + @input_file_name + ":"
1375
+ if @scanner
1376
+ prefix << "#{@scanner.line_no}:#{@scanner.char_no}: "
1377
+ end
1378
+ $stderr.puts prefix + msg
1379
+ end
1380
+
1381
+ def error(msg)
1382
+ warn msg
1383
+ end
1384
+
1385
+ def progress(char)
1386
+ unless @options.quiet
1387
+ @progress.print(char)
1388
+ @progress.flush
1389
+ end
1390
+ end
1391
+
1392
+ def add_token_listener(obj)
1393
+ @token_listeners ||= []
1394
+ @token_listeners << obj
1395
+ end
1396
+
1397
+ def remove_token_listener(obj)
1398
+ @token_listeners.delete(obj)
1399
+ end
1400
+
1401
+ def get_tk
1402
+ tk = nil
1403
+ if @tokens.empty?
1404
+ tk = @scanner.token
1405
+ @read.push @scanner.get_read
1406
+ puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
1407
+ else
1408
+ @read.push @unget_read.shift
1409
+ tk = @tokens.shift
1410
+ puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
1411
+ end
1412
+
1413
+ if tk.kind_of?(TkSYMBEG)
1414
+ set_token_position(tk.line_no, tk.char_no)
1415
+ tk1 = get_tk
1416
+ if tk1.kind_of?(TkId) || tk1.kind_of?(TkOp)
1417
+ tk = Token(TkSYMBOL).set_text(":" + tk1.name)
1418
+ # remove the identifier we just read (we're about to
1419
+ # replace it with a symbol)
1420
+ @token_listeners.each do |obj|
1421
+ obj.pop_token
1422
+ end if @token_listeners
1423
+ else
1424
+ warn("':' not followed by identified or operator")
1425
+ tk = tk1
1426
+ end
1427
+ end
1428
+
1429
+ # inform any listeners of our shiny new token
1430
+ @token_listeners.each do |obj|
1431
+ obj.add_token(tk)
1432
+ end if @token_listeners
1433
+
1434
+ tk
1435
+ end
1436
+
1437
+ def peek_tk
1438
+ unget_tk(tk = get_tk)
1439
+ tk
1440
+ end
1441
+
1442
+ def unget_tk(tk)
1443
+ @tokens.unshift tk
1444
+ @unget_read.unshift @read.pop
1445
+
1446
+ # Remove this token from any listeners
1447
+ @token_listeners.each do |obj|
1448
+ obj.pop_token
1449
+ end if @token_listeners
1450
+ end
1451
+
1452
+ def skip_tkspace(skip_nl = true)
1453
+ tokens = []
1454
+ while ((tk = get_tk).kind_of?(TkSPACE) ||
1455
+ (skip_nl && tk.kind_of?(TkNL)))
1456
+ tokens.push tk
1457
+ end
1458
+ unget_tk(tk)
1459
+ tokens
1460
+ end
1461
+
1462
+ def get_tkread
1463
+ read = @read.join("")
1464
+ @read = []
1465
+ read
1466
+ end
1467
+
1468
+ NORMAL = "::"
1469
+ SINGLE = "<<"
1470
+
1471
+ # Look for the first comment in a file that isn't
1472
+ # a shebang line.
1473
+
1474
+ def collect_first_comment
1475
+ skip_tkspace
1476
+ res = ''
1477
+ first_line = true
1478
+
1479
+ tk = get_tk
1480
+ while tk.kind_of?(TkCOMMENT)
1481
+ if first_line && tk.text[0,2] == "#!"
1482
+ skip_tkspace
1483
+ tk = get_tk
1484
+ else
1485
+ res << tk.text << "\n"
1486
+ tk = get_tk
1487
+ if tk.kind_of? TkNL
1488
+ skip_tkspace(false)
1489
+ tk = get_tk
1490
+ end
1491
+ end
1492
+ first_line = false
1493
+ end
1494
+ unget_tk(tk)
1495
+ res
1496
+ end
1497
+
1498
+ def parse_statements(container, single=NORMAL, current_method = nil)
1499
+ nest = 1
1500
+
1501
+ if container.kind_of?(TopLevel)
1502
+ comment = collect_first_comment
1503
+ look_for_directives_in(container, comment)
1504
+ container.comment = comment unless comment.empty?
1505
+ else
1506
+ comment = ''
1507
+ end
1508
+
1509
+ keep_comment = false
1510
+ while tk = get_tk
1511
+ blank_line_seen = true
1512
+ while tk.kind_of?(TkNL)
1513
+ skip_tkspace(false)
1514
+ if peek_tk.kind_of?(TkCOMMENT)
1515
+ if blank_line_seen
1516
+ comment = ''
1517
+ blank_line_seen = false
1518
+ end
1519
+ tk = get_tk
1520
+ comment << tk.text << "\n"
1521
+ else
1522
+ blank_line_seen = true
1523
+ end
1524
+ tk = get_tk
1525
+ end
1526
+
1527
+ look_for_directives_in(container, comment) unless comment.empty?
1528
+
1529
+ case tk
1530
+
1531
+ when TkCLASS
1532
+ if container.document_children
1533
+ parse_class(container, single, tk, comment)
1534
+ else
1535
+ nest += 1
1536
+ end
1537
+
1538
+ when TkMODULE
1539
+ if container.document_children
1540
+ parse_module(container, single, tk, comment)
1541
+ else
1542
+ nest += 1
1543
+ end
1544
+
1545
+ when TkDEF
1546
+ if container.document_self
1547
+ parse_method(container, single, tk, comment)
1548
+ else
1549
+ nest += 1
1550
+ end
1551
+
1552
+ when TkALIAS
1553
+ if container.document_self
1554
+ parse_alias(container, single, tk, comment)
1555
+ end
1556
+
1557
+ when TkYIELD
1558
+ if current_method.nil?
1559
+ warn("Warning: yield outside of method") if container.document_self
1560
+ else
1561
+ parse_yield(container, single, tk, current_method)
1562
+ end
1563
+
1564
+ # Until and While can have a 'do', which shouldn't increas
1565
+ # the nesting. We can't solve the general case, but we can
1566
+ # handle most occurrences by ignoring a do at the end of a line
1567
+
1568
+ when TkUNTIL, TkWHILE
1569
+ nest += 1
1570
+ puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
1571
+ "line #{tk.line_no}" if $DEBUG
1572
+ skip_optional_do_after_expression
1573
+
1574
+ when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN, TkFOR
1575
+ nest += 1
1576
+ puts "Found #{tk.class} in #{container.name}, nest = #{nest}, " +
1577
+ "line #{tk.line_no}" if $DEBUG
1578
+
1579
+ when TkIDENTIFIER
1580
+ if nest == 1 and current_method.nil?
1581
+ case tk.name
1582
+ when "private", "protected", "public"
1583
+ parse_visibility(container, single, tk)
1584
+ keep_comment = true
1585
+ when "attr"
1586
+ parse_attr(container, single, tk, comment)
1587
+ when /^attr_(reader|writer|accessor)$/
1588
+ parse_attr_accessor(container, single, tk, comment)
1589
+ end
1590
+ end
1591
+
1592
+ case tk.name
1593
+ when "require"
1594
+ parse_require(container, comment)
1595
+ when "include"
1596
+ parse_include(container, comment)
1597
+ end
1598
+
1599
+
1600
+ when TkEND
1601
+ nest -= 1
1602
+ puts "Found 'end' in #{container.name}, nest = #{nest}, line #{tk.line_no}" if $DEBUG
1603
+ puts "Method = #{current_method.name}" if $DEBUG and current_method
1604
+ return if nest == 0
1605
+
1606
+ when TkCOMMENT
1607
+ #puts "Comment: #{tk.text}"
1608
+
1609
+ end
1610
+
1611
+ begin
1612
+ get_tkread
1613
+ skip_tkspace(false)
1614
+ end while peek_tk == TkNL
1615
+
1616
+ comment = '' unless keep_comment
1617
+ keep_comment = false
1618
+ end
1619
+ end
1620
+
1621
+ def parse_class(container, single, tk, comment, &block)
1622
+ progress("c")
1623
+ skip_tkspace
1624
+ case name_t = get_tk
1625
+ when TkCONSTANT
1626
+ name = name_t.name
1627
+ superclass = "Object"
1628
+
1629
+ skip_tkspace(false)
1630
+ if peek_tk.kind_of?(TkLT)
1631
+ get_tk
1632
+ skip_tkspace(true)
1633
+ superclass = get_class_specification
1634
+ superclass = "<unknown>" if superclass.empty?
1635
+ end
1636
+
1637
+ if single == SINGLE
1638
+ cls_type = SingleClass
1639
+ else
1640
+ cls_type = NormalClass
1641
+ end
1642
+
1643
+ cls = container.add_class(cls_type, name, superclass)
1644
+ read_documentation_modifiers(cls, CLASS_MODIFIERS)
1645
+ cls.record_location(@top_level)
1646
+ parse_statements(cls)
1647
+ cls.comment = comment
1648
+
1649
+ when TkLSHFT
1650
+ skip_tkspace
1651
+ case name = get_class_specification
1652
+ when "self"
1653
+ parse_statements(container, SINGLE, &block)
1654
+ else
1655
+
1656
+ # Special case: class << X inside class 'X' adds singleton methods
1657
+ if name == container.name
1658
+ parse_statements(container, SINGLE, &block)
1659
+ else
1660
+ other = TopLevel.find_class_named(name) || Context.new
1661
+ parse_statements(other, SINGLE, &block)
1662
+ end
1663
+ end
1664
+
1665
+ else
1666
+ warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
1667
+ end
1668
+ end
1669
+
1670
+ def parse_module(container, single, tk, comment)
1671
+ progress("m")
1672
+ skip_tkspace
1673
+ name = get_tk.name
1674
+ mod = container.add_module(NormalModule, name)
1675
+ mod.record_location(@top_level)
1676
+ read_documentation_modifiers(mod, CLASS_MODIFIERS)
1677
+ parse_statements(mod)
1678
+ mod.comment = comment
1679
+ end
1680
+
1681
+ def parse_method(container, single, tk, comment)
1682
+ progress(".")
1683
+ line_no = tk.line_no
1684
+ column = tk.char_no
1685
+
1686
+ start_collecting_tokens
1687
+ add_token(tk)
1688
+ add_token_listener(self)
1689
+
1690
+ @scanner.instance_eval{@lex_state = EXPR_FNAME}
1691
+ skip_tkspace(false)
1692
+ name_t = get_tk
1693
+ back_tk = skip_tkspace
1694
+ meth = nil
1695
+
1696
+ dot = get_tk
1697
+ if dot.kind_of?(TkDOT) or dot.kind_of?(TkCOLON2)
1698
+ @scanner.instance_eval{@lex_state = EXPR_FNAME}
1699
+ skip_tkspace
1700
+ name_t2 = get_tk
1701
+ case name_t
1702
+ when TkSELF
1703
+ name = name_t2.name
1704
+ when TkId
1705
+ name = name_t2.name
1706
+ else
1707
+ warn("Unexpected token '#{name_t2.inspect}'")
1708
+ break
1709
+ end
1710
+ meth = AnyMethod.new(get_tkread, name)
1711
+ meth.singleton = true
1712
+ else
1713
+ unget_tk dot
1714
+ back_tk.reverse_each do
1715
+ |tk|
1716
+ unget_tk tk
1717
+ end
1718
+ name = name_t.name
1719
+
1720
+ meth = AnyMethod.new(get_tkread, name)
1721
+ meth.singleton = (single == SINGLE)
1722
+ end
1723
+
1724
+ remove_token_listener(self)
1725
+
1726
+ meth.start_collecting_tokens
1727
+ indent = TkSPACE.new(1,1)
1728
+ indent.set_text(" " * column)
1729
+
1730
+ meth.add_tokens([TkCOMMENT.new(line_no,
1731
+ 1,
1732
+ "# File #{@top_level.file_absolute_name}, line #{line_no}"),
1733
+ NEWLINE_TOKEN,
1734
+ indent])
1735
+
1736
+ meth.add_tokens(@token_stream)
1737
+
1738
+ add_token_listener(meth)
1739
+
1740
+ @scanner.instance_eval{@continue = false}
1741
+ parse_method_parameters(meth)
1742
+
1743
+ if meth.document_self
1744
+ container.add_method(meth)
1745
+ end
1746
+
1747
+ # Having now read the method parameters and documentation modifiers, we
1748
+ # now know whether we have to rename #initialize to ::new
1749
+
1750
+ if name == "initialize" && !meth.singleton
1751
+ if meth.dont_rename_initialize
1752
+ meth.visibility = :protected
1753
+ else
1754
+ meth.singleton = true
1755
+ meth.name = "new"
1756
+ meth.visibility = :public
1757
+ end
1758
+ end
1759
+
1760
+ parse_statements(container, single, meth)
1761
+
1762
+ remove_token_listener(meth)
1763
+
1764
+ meth.comment = comment
1765
+ end
1766
+
1767
+ # Capture the method's parameters. Along the way,
1768
+ # look for a comment containing
1769
+ #
1770
+ # # yields: ....
1771
+ #
1772
+ # and add this as the block_params for the method
1773
+
1774
+ def parse_method_parameters(method)
1775
+ res = parse_method_or_yield_parameters(method)
1776
+ method.params = res unless method.params
1777
+ if method.block_params.nil?
1778
+ read_documentation_modifiers(method, METHOD_MODIFIERS)
1779
+ end
1780
+ end
1781
+
1782
+ def parse_method_or_yield_parameters(method=nil, modifiers=METHOD_MODIFIERS)
1783
+ skip_tkspace(false)
1784
+ tk = get_tk
1785
+
1786
+ # Little hack going on here. In the statement
1787
+ # f = 2*(1+yield)
1788
+ # We see the RPAREN as the next token, so we need
1789
+ # so exit early. THis still won't catch all cases
1790
+ # (such as "a = yield + 1"
1791
+ end_token = case tk
1792
+ when TkLPAREN, TkfLPAREN
1793
+ TkRPAREN
1794
+ when TkRPAREN
1795
+ return ""
1796
+ else
1797
+ TkNL
1798
+ end
1799
+ nest = 0
1800
+
1801
+ loop do
1802
+ puts("Param: #{tk}, #{@scanner.continue} " +
1803
+ "#{@scanner.lex_state} #{nest}") if $DEBUG
1804
+ case tk
1805
+ when TkSEMICOLON
1806
+ break
1807
+ when TkLPAREN, TkfLPAREN
1808
+ nest += 1
1809
+ when end_token
1810
+ if end_token == TkRPAREN
1811
+ nest -= 1
1812
+ break if @scanner.lex_state == EXPR_END and nest <= 0
1813
+ else
1814
+ break unless @scanner.continue
1815
+ end
1816
+ when method && method.block_params.nil? && TkCOMMENT
1817
+ unget_tk(tk)
1818
+ read_documentation_modifiers(method, modifiers)
1819
+ end
1820
+ tk = get_tk
1821
+ end
1822
+ res = get_tkread.tr("\n", " ").strip
1823
+ res = "" if res == ";"
1824
+ res
1825
+ end
1826
+
1827
+ # while, until, and for have an optional
1828
+ def skip_optional_do_after_expression
1829
+ skip_tkspace(false)
1830
+ tk = get_tk
1831
+ case tk
1832
+ when TkLPAREN, TkfLPAREN
1833
+ end_token = TkRPAREN
1834
+ else
1835
+ end_token = TkNL
1836
+ end
1837
+
1838
+ nest = 0
1839
+ @scanner.instance_eval{@continue = false}
1840
+
1841
+ loop do
1842
+ puts("\nWhile: #{tk}, #{@scanner.continue} " +
1843
+ "#{@scanner.lex_state} #{nest}") if $DEBUG
1844
+ case tk
1845
+ when TkSEMICOLON
1846
+ break
1847
+ when TkLPAREN, TkfLPAREN
1848
+ nest += 1
1849
+ when TkDO
1850
+ break if nest.zero?
1851
+ when end_token
1852
+ if end_token == TkRPAREN
1853
+ nest -= 1
1854
+ break if @scanner.lex_state == EXPR_END and nest.zero?
1855
+ else
1856
+ break unless @scanner.continue
1857
+ end
1858
+ end
1859
+ tk = get_tk
1860
+ end
1861
+ skip_tkspace(false)
1862
+ if peek_tk.kind_of? TkDO
1863
+ get_tk
1864
+ end
1865
+ end
1866
+
1867
+ # Return a superclass, which can be either a constant
1868
+ # of an expression
1869
+
1870
+ def get_class_specification
1871
+ tk = get_tk
1872
+ return "self" if tk.kind_of?(TkSELF)
1873
+
1874
+ res = ""
1875
+ while tk.kind_of?(TkCOLON2) ||
1876
+ tk.kind_of?(TkCOLON3) ||
1877
+ tk.kind_of?(TkCONSTANT)
1878
+
1879
+ res += tk.text
1880
+ tk = get_tk
1881
+ end
1882
+
1883
+ unget_tk(tk)
1884
+ skip_tkspace(false)
1885
+
1886
+ get_tkread # empty out read buffer
1887
+
1888
+ tk = get_tk
1889
+
1890
+ case tk
1891
+ when TkNL, TkCOMMENT, TkSEMICOLON
1892
+ unget_tk(tk)
1893
+ return res
1894
+ end
1895
+
1896
+ res += parse_call_parameters(tk)
1897
+ res
1898
+ end
1899
+
1900
+ def parse_call_parameters(tk)
1901
+
1902
+ end_token = case tk
1903
+ when TkLPAREN, TkfLPAREN
1904
+ TkRPAREN
1905
+ when TkRPAREN
1906
+ return ""
1907
+ else
1908
+ TkNL
1909
+ end
1910
+ nest = 0
1911
+
1912
+ loop do
1913
+ puts("Call param: #{tk}, #{@scanner.continue} " +
1914
+ "#{@scanner.lex_state} #{nest}") if $DEBUG
1915
+ case tk
1916
+ when TkSEMICOLON
1917
+ break
1918
+ when TkLPAREN, TkfLPAREN
1919
+ nest += 1
1920
+ when end_token
1921
+ if end_token == TkRPAREN
1922
+ nest -= 1
1923
+ break if @scanner.lex_state == EXPR_END and nest <= 0
1924
+ else
1925
+ break unless @scanner.continue
1926
+ end
1927
+ when TkCOMMENT
1928
+ unget_tk(tk)
1929
+ break
1930
+ end
1931
+ tk = get_tk
1932
+ end
1933
+ res = get_tkread.tr("\n", " ").strip
1934
+ res = "" if res == ";"
1935
+ res
1936
+ end
1937
+
1938
+
1939
+ # Parse a constant, which might be qualified by
1940
+ # one or more class or module names
1941
+
1942
+ def get_constant
1943
+ res = ""
1944
+ skip_tkspace(false)
1945
+ tk = get_tk
1946
+
1947
+ while tk.kind_of?(TkCOLON2) ||
1948
+ tk.kind_of?(TkCOLON3) ||
1949
+ tk.kind_of?(TkCONSTANT)
1950
+
1951
+ res += tk.text
1952
+ tk = get_tk
1953
+ end
1954
+
1955
+ # if res.empty?
1956
+ # warn("Unexpected token #{tk} in constant")
1957
+ # end
1958
+ unget_tk(tk)
1959
+ res
1960
+ end
1961
+
1962
+ # Get a constant that may be surrounded by parens
1963
+
1964
+ def get_constant_with_optional_parens
1965
+ skip_tkspace(false)
1966
+ nest = 0
1967
+ while (tk = peek_tk).kind_of?(TkLPAREN) || tk.kind_of?(TkfLPAREN)
1968
+ get_tk
1969
+ skip_tkspace(true)
1970
+ nest += 1
1971
+ end
1972
+
1973
+ name = get_constant
1974
+
1975
+ while nest > 0
1976
+ skip_tkspace(true)
1977
+ tk = get_tk
1978
+ warn("')' expected") unless tk.kind_of?(TkRPAREN)
1979
+ nest -= 1
1980
+ end
1981
+ name
1982
+ end
1983
+
1984
+ # Directives are modifier comments that can appear after class, module,
1985
+ # or method names. For example
1986
+ #
1987
+ # def fred # :yields: a, b
1988
+ #
1989
+ # or
1990
+ #
1991
+ # class SM # :nodoc:
1992
+ #
1993
+ # we return the directive name and any parameters as a two element array
1994
+
1995
+ def read_directive(allowed)
1996
+ skip_tkspace(false)
1997
+ tk = get_tk
1998
+ puts "directive: #{tk.inspect}" if $DEBUG
1999
+ result = nil
2000
+ if tk.kind_of?(TkCOMMENT)
2001
+ if tk.text =~ /\s*:?(\w+):\s*(.*)/
2002
+ directive = $1.downcase
2003
+ if allowed.include?(directive)
2004
+ result = [directive, $2]
2005
+ end
2006
+ end
2007
+ else
2008
+ unget_tk(tk)
2009
+ end
2010
+ result
2011
+ end
2012
+
2013
+
2014
+ def read_documentation_modifiers(context, allow)
2015
+ dir = read_directive(allow)
2016
+
2017
+ case dir[0]
2018
+
2019
+ when "notnew", "not_new", "not-new"
2020
+ context.dont_rename_initialize = true
2021
+
2022
+ when "nodoc"
2023
+ context.document_self = false
2024
+ if dir[1].downcase == "all"
2025
+ context.document_children = false
2026
+ end
2027
+
2028
+ when "doc"
2029
+ context.document_self = true
2030
+ context.force_documentation = true
2031
+
2032
+ when "yield", "yields"
2033
+ unless context.params.nil?
2034
+ context.params.sub!(/(,|)\s*&\w+/,'') # remove parameter &proc
2035
+ end
2036
+ context.block_params = dir[1]
2037
+
2038
+ when "arg", "args"
2039
+ context.params = dir[1]
2040
+ end if dir
2041
+ end
2042
+
2043
+
2044
+ # Look for directives in a normal comment block:
2045
+ #
2046
+ # #-- - don't display comment from this point forward
2047
+ # #** - Don't display this comment line
2048
+ #
2049
+ #
2050
+ # This routine modifies it's parameter
2051
+
2052
+ def look_for_directives_in(context, comment)
2053
+
2054
+ preprocess = SM::PreProcess.new(@input_file_name,
2055
+ @options.rdoc_include)
2056
+
2057
+ preprocess.handle(comment) do |directive, param|
2058
+
2059
+ case directive
2060
+ when "nodoc"
2061
+ context.document_self = false
2062
+ ""
2063
+ when "doc"
2064
+ context.document_self = true
2065
+ context.force_documentation = true
2066
+ ""
2067
+
2068
+ when "main"
2069
+ options = Options.instance
2070
+ options.main_page = param
2071
+
2072
+ when "title"
2073
+ options = Options.instance
2074
+ options.title = param
2075
+ ""
2076
+
2077
+ else
2078
+ warn "Unrecognized directive '#{directive}'"
2079
+ break
2080
+ end
2081
+ end
2082
+
2083
+ remove_private_comments(comment)
2084
+ end
2085
+
2086
+ def remove_private_comments(comment)
2087
+ comment.gsub!(/^#--.*?^#\+\+/m, '')
2088
+ comment.sub!(/^#--.*/m, '')
2089
+ comment.gsub!(/^#\*\*.*\n/, '')
2090
+ end
2091
+
2092
+
2093
+
2094
+ def get_symbol_or_name
2095
+ tk = get_tk
2096
+ case tk
2097
+ when TkSYMBOL
2098
+ tk.text.sub(/^:/, '')
2099
+ when TkId, TkOp
2100
+ tk.name
2101
+ else
2102
+ raise "Name or symbol expected (got #{tk})"
2103
+ end
2104
+ end
2105
+
2106
+ def parse_alias(context, single, tk, comment)
2107
+ skip_tkspace
2108
+ new_name = get_symbol_or_name
2109
+ @scanner.instance_eval{@lex_state = EXPR_FNAME}
2110
+ skip_tkspace
2111
+ old_name = get_symbol_or_name
2112
+ al = Alias.new(get_tkread, old_name, new_name, comment)
2113
+ read_documentation_modifiers(al, ATTR_MODIFIERS)
2114
+ if al.document_self
2115
+ context.add_alias(al)
2116
+ end
2117
+ end
2118
+
2119
+ def parse_yield_parameters
2120
+ parse_method_or_yield_parameters
2121
+ end
2122
+
2123
+ def parse_yield(context, single, tk, method)
2124
+ if method.block_params.nil?
2125
+ get_tkread
2126
+ @scanner.instance_eval{@continue = false}
2127
+ method.block_params = parse_yield_parameters
2128
+ end
2129
+ end
2130
+
2131
+ def parse_require(context, comment)
2132
+ skip_tkspace_comment
2133
+ tk = get_tk
2134
+ if tk.kind_of? TkLPAREN
2135
+ skip_tkspace_comment
2136
+ tk = get_tk
2137
+ end
2138
+
2139
+ name = nil
2140
+ case tk
2141
+ when TkSTRING
2142
+ name = tk.text
2143
+ when TkCONSTANT, TkIDENTIFIER, TkIVAR, TkGVAR
2144
+ name = tk.name
2145
+ when TkDSTRING
2146
+ warn "Skipping require of dynamic string: #{tk.text}"
2147
+ else
2148
+ error("Unknown argument type to require: #{tk}")
2149
+ end
2150
+ if name
2151
+ context.add_require(Require.new(name, comment))
2152
+ end
2153
+ end
2154
+
2155
+ def parse_include(context, comment)
2156
+ skip_tkspace_comment
2157
+ name = get_constant_with_optional_parens
2158
+ unless name.empty?
2159
+ context.add_include(Include.new(name, comment))
2160
+ end
2161
+ end
2162
+
2163
+ def get_bool
2164
+ skip_tkspace
2165
+ tk = get_tk
2166
+ case tk
2167
+ when TkTRUE
2168
+ true
2169
+ when TkFALSE, TkNIL
2170
+ false
2171
+ else
2172
+ unget_tk tk
2173
+ true
2174
+ end
2175
+ end
2176
+
2177
+ def parse_attr(context, single, tk, comment)
2178
+ args = parse_symbol_arg(1)
2179
+ if args.size > 0
2180
+ name = args[0]
2181
+ rw = "R"
2182
+ skip_tkspace(false)
2183
+ tk = get_tk
2184
+ if tk.kind_of? TkCOMMA
2185
+ rw = "RW" if get_bool
2186
+ else
2187
+ unget_tk tk
2188
+ end
2189
+ att = Attr.new(get_tkread, name, rw, comment)
2190
+ read_documentation_modifiers(att, ATTR_MODIFIERS)
2191
+ if att.document_self
2192
+ context.add_attribute(att)
2193
+ end
2194
+ else
2195
+ error("Missing argument to 'attr'")
2196
+ end
2197
+
2198
+ end
2199
+
2200
+ def parse_visibility(container, single, tk)
2201
+ vis = case tk.name
2202
+ when "private" then :private
2203
+ when "protected" then :protected
2204
+ when "public" then :public
2205
+ else raise "Invalid visibility: #{tk.name}"
2206
+ end
2207
+
2208
+ skip_tkspace_comment(false)
2209
+ if peek_tk.kind_of? TkNL
2210
+ container.ongoing_visibility = vis
2211
+ else
2212
+ args = parse_symbol_arg
2213
+ container.set_visibility_for(args, vis)
2214
+ end
2215
+ end
2216
+
2217
+ def parse_attr_accessor(context, single, tk, comment)
2218
+ args = parse_symbol_arg
2219
+ read = get_tkread
2220
+ rw = "?"
2221
+
2222
+ # If nodoc is given, don't document any of them
2223
+
2224
+ tmp = CodeObject.new
2225
+ read_documentation_modifiers(tmp, ATTR_MODIFIERS)
2226
+ return unless tmp.document_self
2227
+
2228
+ case tk.name
2229
+ when "attr_reader" then rw = "R"
2230
+ when "attr_writer" then rw = "W"
2231
+ when "attr_accessor" then rw = "RW"
2232
+ end
2233
+
2234
+ for name in args
2235
+ att = Attr.new(get_tkread, name, rw, comment)
2236
+ context.add_attribute(att)
2237
+ end
2238
+ end
2239
+
2240
+ def skip_tkspace_comment(skip_nl = true)
2241
+ loop do
2242
+ skip_tkspace(skip_nl)
2243
+ return unless peek_tk.kind_of? TkCOMMENT
2244
+ get_tk
2245
+ end
2246
+ end
2247
+
2248
+ def parse_symbol_arg(no = nil)
2249
+
2250
+ args = []
2251
+ skip_tkspace_comment
2252
+ case tk = get_tk
2253
+ when TkLPAREN
2254
+ loop do
2255
+ skip_tkspace_comment
2256
+ if tk1 = parse_symbol_in_arg
2257
+ args.push tk1
2258
+ break if no and args.size >= no
2259
+ end
2260
+
2261
+ skip_tkspace_comment
2262
+ case tk2 = get_tk
2263
+ when TkRPAREN
2264
+ break
2265
+ when TkCOMMA
2266
+ else
2267
+ warn("unexpected token: '#{tk.inspect}'")
2268
+ break
2269
+ end
2270
+ end
2271
+ else
2272
+ unget_tk tk
2273
+ if tk = parse_symbol_in_arg
2274
+ args.push tk
2275
+ return args if no and args.size >= no
2276
+ end
2277
+
2278
+ loop do
2279
+ # skip_tkspace_comment(false)
2280
+ skip_tkspace(false)
2281
+
2282
+ tk1 = get_tk
2283
+ unless tk1.kind_of?(TkCOMMA)
2284
+ unget_tk tk1
2285
+ break
2286
+ end
2287
+
2288
+ skip_tkspace_comment
2289
+ if tk = parse_symbol_in_arg
2290
+ args.push tk
2291
+ break if no and args.size >= no
2292
+ end
2293
+ end
2294
+ end
2295
+ args
2296
+ end
2297
+
2298
+ def parse_symbol_in_arg
2299
+ case tk = get_tk
2300
+ when TkSYMBOL
2301
+ tk.text.sub(/^:/, '')
2302
+ when TkSTRING
2303
+ eval @read[-1]
2304
+ else
2305
+ warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG
2306
+ nil
2307
+ end
2308
+ end
2309
+ end
2310
+
2311
+ end