gitlab-pygments.rb 0.3.2 → 0.5.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (199) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/CHANGELOG.md +71 -0
  4. data/Gemfile +1 -1
  5. data/LICENSE +17 -0
  6. data/README.md +28 -1
  7. data/Rakefile +2 -5
  8. data/lexers +0 -0
  9. data/lib/pygments/mentos.py +19 -12
  10. data/lib/pygments/popen.rb +21 -4
  11. data/lib/pygments/version.rb +1 -1
  12. data/pygments.rb.gemspec +2 -0
  13. data/test/test_pygments.rb +13 -2
  14. data/vendor/custom_lexers/github.py +206 -3
  15. data/vendor/pygments-main/AUTHORS +41 -3
  16. data/vendor/pygments-main/CHANGES +132 -5
  17. data/vendor/pygments-main/LICENSE +1 -1
  18. data/vendor/pygments-main/Makefile +1 -1
  19. data/vendor/pygments-main/REVISION +1 -1
  20. data/vendor/pygments-main/docs/generate.py +1 -1
  21. data/vendor/pygments-main/docs/src/api.txt +1 -1
  22. data/vendor/pygments-main/docs/src/index.txt +1 -1
  23. data/vendor/pygments-main/docs/src/integrate.txt +5 -0
  24. data/vendor/pygments-main/docs/src/java.txt +70 -0
  25. data/vendor/pygments-main/docs/src/lexerdevelopment.txt +52 -0
  26. data/vendor/pygments-main/external/autopygmentize +64 -0
  27. data/vendor/pygments-main/external/lasso-builtins-generator-9.lasso +144 -0
  28. data/vendor/pygments-main/external/markdown-processor.py +2 -2
  29. data/vendor/pygments-main/external/moin-parser.py +1 -1
  30. data/vendor/pygments-main/external/rst-directive-old.py +1 -1
  31. data/vendor/pygments-main/external/rst-directive.py +1 -1
  32. data/vendor/pygments-main/pygmentize +1 -1
  33. data/vendor/pygments-main/pygments/__init__.py +2 -2
  34. data/vendor/pygments-main/pygments/cmdline.py +14 -6
  35. data/vendor/pygments-main/pygments/console.py +1 -1
  36. data/vendor/pygments-main/pygments/filter.py +1 -1
  37. data/vendor/pygments-main/pygments/filters/__init__.py +3 -4
  38. data/vendor/pygments-main/pygments/formatter.py +4 -1
  39. data/vendor/pygments-main/pygments/formatters/__init__.py +1 -1
  40. data/vendor/pygments-main/pygments/formatters/_mapping.py +3 -3
  41. data/vendor/pygments-main/pygments/formatters/bbcode.py +1 -1
  42. data/vendor/pygments-main/pygments/formatters/html.py +73 -2
  43. data/vendor/pygments-main/pygments/formatters/img.py +2 -2
  44. data/vendor/pygments-main/pygments/formatters/latex.py +9 -9
  45. data/vendor/pygments-main/pygments/formatters/other.py +1 -3
  46. data/vendor/pygments-main/pygments/formatters/rtf.py +1 -1
  47. data/vendor/pygments-main/pygments/formatters/svg.py +1 -1
  48. data/vendor/pygments-main/pygments/formatters/terminal.py +1 -1
  49. data/vendor/pygments-main/pygments/formatters/terminal256.py +1 -1
  50. data/vendor/pygments-main/pygments/lexer.py +78 -10
  51. data/vendor/pygments-main/pygments/lexers/__init__.py +13 -2
  52. data/vendor/pygments-main/pygments/lexers/_asybuiltins.py +1 -1
  53. data/vendor/pygments-main/pygments/lexers/_clbuiltins.py +1 -1
  54. data/vendor/pygments-main/pygments/lexers/_lassobuiltins.py +5172 -0
  55. data/vendor/pygments-main/pygments/lexers/_luabuiltins.py +1 -1
  56. data/vendor/pygments-main/pygments/lexers/_mapping.py +92 -36
  57. data/vendor/pygments-main/pygments/lexers/_openedgebuiltins.py +562 -0
  58. data/vendor/pygments-main/pygments/lexers/_phpbuiltins.py +2 -2
  59. data/vendor/pygments-main/pygments/lexers/_postgres_builtins.py +4 -3
  60. data/vendor/pygments-main/pygments/lexers/_robotframeworklexer.py +557 -0
  61. data/vendor/pygments-main/pygments/lexers/_scilab_builtins.py +11 -0
  62. data/vendor/pygments-main/pygments/lexers/_sourcemodbuiltins.py +1072 -0
  63. data/vendor/pygments-main/pygments/lexers/_stan_builtins.py +360 -0
  64. data/vendor/pygments-main/pygments/lexers/_vimbuiltins.py +13 -3
  65. data/vendor/pygments-main/pygments/lexers/agile.py +525 -38
  66. data/vendor/pygments-main/pygments/lexers/asm.py +45 -7
  67. data/vendor/pygments-main/pygments/lexers/compiled.py +1257 -425
  68. data/vendor/pygments-main/pygments/lexers/dalvik.py +104 -0
  69. data/vendor/pygments-main/pygments/lexers/dotnet.py +97 -62
  70. data/vendor/pygments-main/pygments/lexers/foxpro.py +428 -0
  71. data/vendor/pygments-main/pygments/lexers/functional.py +931 -32
  72. data/vendor/pygments-main/pygments/lexers/github.py +206 -3
  73. data/vendor/pygments-main/pygments/lexers/hdl.py +3 -3
  74. data/vendor/pygments-main/pygments/lexers/jvm.py +309 -44
  75. data/vendor/pygments-main/pygments/lexers/math.py +876 -30
  76. data/vendor/pygments-main/pygments/lexers/other.py +956 -517
  77. data/vendor/pygments-main/pygments/lexers/parsers.py +85 -2
  78. data/vendor/pygments-main/pygments/lexers/shell.py +81 -18
  79. data/vendor/pygments-main/pygments/lexers/special.py +1 -1
  80. data/vendor/pygments-main/pygments/lexers/sql.py +2 -2
  81. data/vendor/pygments-main/pygments/lexers/templates.py +119 -8
  82. data/vendor/pygments-main/pygments/lexers/text.py +155 -15
  83. data/vendor/pygments-main/pygments/lexers/web.py +1578 -397
  84. data/vendor/pygments-main/pygments/modeline.py +40 -0
  85. data/vendor/pygments-main/pygments/plugin.py +1 -1
  86. data/vendor/pygments-main/pygments/scanner.py +1 -1
  87. data/vendor/pygments-main/pygments/style.py +1 -1
  88. data/vendor/pygments-main/pygments/styles/__init__.py +1 -1
  89. data/vendor/pygments-main/pygments/styles/autumn.py +1 -1
  90. data/vendor/pygments-main/pygments/styles/borland.py +1 -1
  91. data/vendor/pygments-main/pygments/styles/bw.py +1 -1
  92. data/vendor/pygments-main/pygments/styles/colorful.py +1 -1
  93. data/vendor/pygments-main/pygments/styles/default.py +1 -1
  94. data/vendor/pygments-main/pygments/styles/emacs.py +1 -1
  95. data/vendor/pygments-main/pygments/styles/friendly.py +1 -1
  96. data/vendor/pygments-main/pygments/styles/fruity.py +1 -1
  97. data/vendor/pygments-main/pygments/styles/manni.py +1 -1
  98. data/vendor/pygments-main/pygments/styles/monokai.py +1 -1
  99. data/vendor/pygments-main/pygments/styles/murphy.py +1 -1
  100. data/vendor/pygments-main/pygments/styles/native.py +1 -1
  101. data/vendor/pygments-main/pygments/styles/pastie.py +1 -1
  102. data/vendor/pygments-main/pygments/styles/perldoc.py +1 -1
  103. data/vendor/pygments-main/pygments/styles/rrt.py +1 -1
  104. data/vendor/pygments-main/pygments/styles/tango.py +1 -1
  105. data/vendor/pygments-main/pygments/styles/trac.py +1 -1
  106. data/vendor/pygments-main/pygments/styles/vim.py +1 -1
  107. data/vendor/pygments-main/pygments/styles/vs.py +1 -1
  108. data/vendor/pygments-main/pygments/token.py +1 -1
  109. data/vendor/pygments-main/pygments/unistring.py +36 -26
  110. data/vendor/pygments-main/pygments/util.py +46 -1
  111. data/vendor/pygments-main/scripts/check_sources.py +2 -2
  112. data/vendor/pygments-main/scripts/detect_missing_analyse_text.py +2 -0
  113. data/vendor/pygments-main/scripts/find_codetags.py +1 -1
  114. data/vendor/pygments-main/scripts/find_error.py +5 -6
  115. data/vendor/pygments-main/setup.cfg +1 -0
  116. data/vendor/pygments-main/setup.py +6 -4
  117. data/vendor/pygments-main/tests/examplefiles/BOM.js +1 -0
  118. data/vendor/pygments-main/tests/examplefiles/Config.in.cache +1973 -0
  119. data/vendor/pygments-main/tests/examplefiles/Deflate.fs +578 -0
  120. data/vendor/pygments-main/tests/examplefiles/Get-CommandDefinitionHtml.ps1 +66 -0
  121. data/vendor/pygments-main/tests/examplefiles/IPDispatchC.nc +104 -0
  122. data/vendor/pygments-main/tests/examplefiles/IPDispatchP.nc +671 -0
  123. data/vendor/pygments-main/tests/examplefiles/RoleQ.pm6 +23 -0
  124. data/vendor/pygments-main/tests/examplefiles/autoit_submit.au3 +25 -0
  125. data/vendor/pygments-main/tests/examplefiles/bigtest.nsi +308 -0
  126. data/vendor/pygments-main/tests/examplefiles/ca65_example +284 -0
  127. data/vendor/pygments-main/tests/examplefiles/cbmbas_example +9 -0
  128. data/vendor/pygments-main/tests/examplefiles/classes.dylan +89 -4
  129. data/vendor/pygments-main/tests/examplefiles/example.Rd +78 -0
  130. data/vendor/pygments-main/tests/examplefiles/example.bug +54 -0
  131. data/vendor/pygments-main/tests/examplefiles/example.ceylon +52 -0
  132. data/vendor/pygments-main/tests/examplefiles/example.clay +33 -0
  133. data/vendor/pygments-main/tests/examplefiles/example.cob +3556 -0
  134. data/vendor/pygments-main/tests/examplefiles/example.hx +142 -0
  135. data/vendor/pygments-main/tests/examplefiles/example.jag +48 -0
  136. data/vendor/pygments-main/tests/examplefiles/example.lagda +19 -0
  137. data/vendor/pygments-main/tests/examplefiles/example.monkey +152 -0
  138. data/vendor/pygments-main/tests/examplefiles/example.msc +43 -0
  139. data/vendor/pygments-main/tests/examplefiles/example.prg +161 -0
  140. data/vendor/pygments-main/tests/examplefiles/example.reg +19 -0
  141. data/vendor/pygments-main/tests/examplefiles/example.rexx +50 -0
  142. data/vendor/pygments-main/tests/examplefiles/example.rkt +95 -0
  143. data/vendor/pygments-main/tests/examplefiles/example.rpf +4 -0
  144. data/vendor/pygments-main/tests/examplefiles/example.shell-session +45 -0
  145. data/vendor/pygments-main/tests/examplefiles/example.stan +108 -0
  146. data/vendor/pygments-main/tests/examplefiles/example.ts +28 -0
  147. data/vendor/pygments-main/tests/examplefiles/example.xtend +34 -0
  148. data/vendor/pygments-main/tests/examplefiles/example2.msc +79 -0
  149. data/vendor/pygments-main/tests/examplefiles/garcia-wachs.kk +133 -0
  150. data/vendor/pygments-main/tests/examplefiles/grammar-test.p6 +22 -0
  151. data/vendor/pygments-main/tests/examplefiles/hello.smali +40 -0
  152. data/vendor/pygments-main/tests/examplefiles/hello.sp +9 -0
  153. data/vendor/pygments-main/tests/examplefiles/http_request_example +2 -1
  154. data/vendor/pygments-main/tests/examplefiles/http_response_example +4 -2
  155. data/vendor/pygments-main/tests/examplefiles/inet_pton6.dg +71 -0
  156. data/vendor/pygments-main/tests/examplefiles/json.lasso +301 -0
  157. data/vendor/pygments-main/tests/examplefiles/json.lasso9 +213 -0
  158. data/vendor/pygments-main/tests/examplefiles/livescript-demo.ls +41 -0
  159. data/vendor/pygments-main/tests/examplefiles/logos_example.xm +28 -0
  160. data/vendor/pygments-main/tests/examplefiles/matlab_sample +5 -2
  161. data/vendor/pygments-main/tests/examplefiles/metagrammar.treetop +455 -0
  162. data/vendor/pygments-main/tests/examplefiles/mg_sample.pro +73 -0
  163. data/vendor/pygments-main/tests/examplefiles/minehunt.qml +112 -0
  164. data/vendor/pygments-main/tests/examplefiles/nanomsg.intr +95 -0
  165. data/vendor/pygments-main/tests/examplefiles/objc_example.m +7 -0
  166. data/vendor/pygments-main/tests/examplefiles/phpMyAdmin.spec +163 -0
  167. data/vendor/pygments-main/tests/examplefiles/py3tb_test.py3tb +4 -0
  168. data/vendor/pygments-main/tests/examplefiles/pytb_test3.pytb +4 -0
  169. data/vendor/pygments-main/tests/examplefiles/robotframework.txt +39 -0
  170. data/vendor/pygments-main/tests/examplefiles/rust_example.rs +233 -0
  171. data/vendor/pygments-main/tests/examplefiles/session.dylan-console +9 -0
  172. data/vendor/pygments-main/tests/examplefiles/swig_java.swg +1329 -0
  173. data/vendor/pygments-main/tests/examplefiles/swig_std_vector.i +225 -0
  174. data/vendor/pygments-main/tests/examplefiles/test.R +149 -115
  175. data/vendor/pygments-main/tests/examplefiles/test.agda +102 -0
  176. data/vendor/pygments-main/tests/examplefiles/test.bb +95 -0
  177. data/vendor/pygments-main/tests/examplefiles/test.cu +36 -0
  178. data/vendor/pygments-main/tests/examplefiles/test.ebnf +31 -0
  179. data/vendor/pygments-main/tests/examplefiles/test.opa +10 -0
  180. data/vendor/pygments-main/tests/examplefiles/test.p6 +252 -0
  181. data/vendor/pygments-main/tests/examplefiles/test2.pypylog +120 -0
  182. data/vendor/pygments-main/tests/examplefiles/type.lisp +16 -0
  183. data/vendor/pygments-main/tests/examplefiles/unix-io.lid +37 -0
  184. data/vendor/pygments-main/tests/old_run.py +1 -1
  185. data/vendor/pygments-main/tests/run.py +3 -2
  186. data/vendor/pygments-main/tests/support/tags +36 -0
  187. data/vendor/pygments-main/tests/test_basic_api.py +4 -3
  188. data/vendor/pygments-main/tests/test_clexer.py +1 -1
  189. data/vendor/pygments-main/tests/test_cmdline.py +1 -1
  190. data/vendor/pygments-main/tests/test_examplefiles.py +3 -1
  191. data/vendor/pygments-main/tests/test_html_formatter.py +17 -1
  192. data/vendor/pygments-main/tests/test_latex_formatter.py +1 -1
  193. data/vendor/pygments-main/tests/test_lexers_other.py +68 -0
  194. data/vendor/pygments-main/tests/test_perllexer.py +1 -1
  195. data/vendor/pygments-main/tests/test_regexlexer.py +1 -1
  196. data/vendor/pygments-main/tests/test_token.py +1 -1
  197. data/vendor/pygments-main/tests/test_using_api.py +1 -1
  198. data/vendor/pygments-main/tests/test_util.py +22 -3
  199. metadata +84 -16
@@ -12,7 +12,7 @@
12
12
  internet connection. don't run that at home, use
13
13
  a server ;-)
14
14
 
15
- :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
15
+ :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
16
16
  :license: BSD, see LICENSE for details.
17
17
  """
18
18
 
@@ -3784,4 +3784,4 @@ if __name__ == '__main__':
3784
3784
  regenerate(__file__, modules)
3785
3785
  shutil.rmtree(PHP_MANUAL_DIR)
3786
3786
 
3787
- run()
3787
+ run()
@@ -1,10 +1,11 @@
1
+ # -*- coding: utf-8 -*-
1
2
  """
2
3
  pygments.lexers._postgres_builtins
3
4
  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4
5
 
5
6
  Self-updating data files for PostgreSQL lexer.
6
7
 
7
- :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
8
+ :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
8
9
  :license: BSD, see LICENSE for details.
9
10
  """
10
11
 
@@ -41,7 +42,6 @@ def parse_keywords(f):
41
42
 
42
43
  def parse_datatypes(f):
43
44
  dt = set()
44
- re_entry = re.compile('\s*<entry><type>([^<]+)</type></entry>')
45
45
  for line in f:
46
46
  if '<sect1' in line:
47
47
  break
@@ -57,7 +57,8 @@ def parse_datatypes(f):
57
57
  line = re.sub("<[^>]+>", "", line)
58
58
 
59
59
  # Drop the parts containing braces
60
- for tmp in [ t for tmp in line.split('[') for t in tmp.split(']') if "(" not in t ]:
60
+ for tmp in [t for tmp in line.split('[')
61
+ for t in tmp.split(']') if "(" not in t]:
61
62
  for t in tmp.split(','):
62
63
  t = t.strip()
63
64
  if not t: continue
@@ -0,0 +1,557 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ pygments.lexers._robotframeworklexer
4
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5
+
6
+ Lexer for Robot Framework.
7
+
8
+ :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
9
+ :license: BSD, see LICENSE for details.
10
+ """
11
+
12
+ # Copyright 2012 Nokia Siemens Networks Oyj
13
+ #
14
+ # Licensed under the Apache License, Version 2.0 (the "License");
15
+ # you may not use this file except in compliance with the License.
16
+ # You may obtain a copy of the License at
17
+ #
18
+ # http://www.apache.org/licenses/LICENSE-2.0
19
+ #
20
+ # Unless required by applicable law or agreed to in writing, software
21
+ # distributed under the License is distributed on an "AS IS" BASIS,
22
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23
+ # See the License for the specific language governing permissions and
24
+ # limitations under the License.
25
+
26
+ import re
27
+
28
+ from pygments.lexer import Lexer
29
+ from pygments.token import Token
30
+
31
+
32
+ HEADING = Token.Generic.Heading
33
+ SETTING = Token.Keyword.Namespace
34
+ IMPORT = Token.Name.Namespace
35
+ TC_KW_NAME = Token.Generic.Subheading
36
+ KEYWORD = Token.Name.Function
37
+ ARGUMENT = Token.String
38
+ VARIABLE = Token.Name.Variable
39
+ COMMENT = Token.Comment
40
+ SEPARATOR = Token.Punctuation
41
+ SYNTAX = Token.Punctuation
42
+ GHERKIN = Token.Generic.Emph
43
+ ERROR = Token.Error
44
+
45
+
46
+ def normalize(string, remove=''):
47
+ string = string.lower()
48
+ for char in remove + ' ':
49
+ if char in string:
50
+ string = string.replace(char, '')
51
+ return string
52
+
53
+
54
+ class RobotFrameworkLexer(Lexer):
55
+ """
56
+ For `Robot Framework <http://robotframework.org>`_ test data.
57
+
58
+ Supports both space and pipe separated plain text formats.
59
+
60
+ *New in Pygments 1.6.*
61
+ """
62
+ name = 'RobotFramework'
63
+ aliases = ['RobotFramework', 'robotframework']
64
+ filenames = ['*.txt', '*.robot']
65
+ mimetypes = ['text/x-robotframework']
66
+
67
+ def __init__(self, **options):
68
+ options['tabsize'] = 2
69
+ options['encoding'] = 'UTF-8'
70
+ Lexer.__init__(self, **options)
71
+
72
+ def get_tokens_unprocessed(self, text):
73
+ row_tokenizer = RowTokenizer()
74
+ var_tokenizer = VariableTokenizer()
75
+ index = 0
76
+ for row in text.splitlines():
77
+ for value, token in row_tokenizer.tokenize(row):
78
+ for value, token in var_tokenizer.tokenize(value, token):
79
+ if value:
80
+ yield index, token, unicode(value)
81
+ index += len(value)
82
+
83
+
84
+ class VariableTokenizer(object):
85
+
86
+ def tokenize(self, string, token):
87
+ var = VariableSplitter(string, identifiers='$@%')
88
+ if var.start < 0 or token in (COMMENT, ERROR):
89
+ yield string, token
90
+ return
91
+ for value, token in self._tokenize(var, string, token):
92
+ if value:
93
+ yield value, token
94
+
95
+ def _tokenize(self, var, string, orig_token):
96
+ before = string[:var.start]
97
+ yield before, orig_token
98
+ yield var.identifier + '{', SYNTAX
99
+ for value, token in self.tokenize(var.base, VARIABLE):
100
+ yield value, token
101
+ yield '}', SYNTAX
102
+ if var.index:
103
+ yield '[', SYNTAX
104
+ for value, token in self.tokenize(var.index, VARIABLE):
105
+ yield value, token
106
+ yield ']', SYNTAX
107
+ for value, token in self.tokenize(string[var.end:], orig_token):
108
+ yield value, token
109
+
110
+
111
+ class RowTokenizer(object):
112
+
113
+ def __init__(self):
114
+ self._table = UnknownTable()
115
+ self._splitter = RowSplitter()
116
+ testcases = TestCaseTable()
117
+ settings = SettingTable(testcases.set_default_template)
118
+ variables = VariableTable()
119
+ keywords = KeywordTable()
120
+ self._tables = {'settings': settings, 'setting': settings,
121
+ 'metadata': settings,
122
+ 'variables': variables, 'variable': variables,
123
+ 'testcases': testcases, 'testcase': testcases,
124
+ 'keywords': keywords, 'keyword': keywords,
125
+ 'userkeywords': keywords, 'userkeyword': keywords}
126
+
127
+ def tokenize(self, row):
128
+ commented = False
129
+ heading = False
130
+ for index, value in enumerate(self._splitter.split(row)):
131
+ # First value, and every second after that, is a separator.
132
+ index, separator = divmod(index-1, 2)
133
+ if value.startswith('#'):
134
+ commented = True
135
+ elif index == 0 and value.startswith('*'):
136
+ self._table = self._start_table(value)
137
+ heading = True
138
+ for value, token in self._tokenize(value, index, commented,
139
+ separator, heading):
140
+ yield value, token
141
+ self._table.end_row()
142
+
143
+ def _start_table(self, header):
144
+ name = normalize(header, remove='*')
145
+ return self._tables.get(name, UnknownTable())
146
+
147
+ def _tokenize(self, value, index, commented, separator, heading):
148
+ if commented:
149
+ yield value, COMMENT
150
+ elif separator:
151
+ yield value, SEPARATOR
152
+ elif heading:
153
+ yield value, HEADING
154
+ else:
155
+ for value, token in self._table.tokenize(value, index):
156
+ yield value, token
157
+
158
+
159
+ class RowSplitter(object):
160
+ _space_splitter = re.compile('( {2,})')
161
+ _pipe_splitter = re.compile('((?:^| +)\|(?: +|$))')
162
+
163
+ def split(self, row):
164
+ splitter = (row.startswith('| ') and self._split_from_pipes
165
+ or self._split_from_spaces)
166
+ for value in splitter(row):
167
+ yield value
168
+ yield '\n'
169
+
170
+ def _split_from_spaces(self, row):
171
+ yield '' # Start with (pseudo)separator similarly as with pipes
172
+ for value in self._space_splitter.split(row):
173
+ yield value
174
+
175
+ def _split_from_pipes(self, row):
176
+ _, separator, rest = self._pipe_splitter.split(row, 1)
177
+ yield separator
178
+ while self._pipe_splitter.search(rest):
179
+ cell, separator, rest = self._pipe_splitter.split(rest, 1)
180
+ yield cell
181
+ yield separator
182
+ yield rest
183
+
184
+
185
+ class Tokenizer(object):
186
+ _tokens = None
187
+
188
+ def __init__(self):
189
+ self._index = 0
190
+
191
+ def tokenize(self, value):
192
+ values_and_tokens = self._tokenize(value, self._index)
193
+ self._index += 1
194
+ if isinstance(values_and_tokens, type(Token)):
195
+ values_and_tokens = [(value, values_and_tokens)]
196
+ return values_and_tokens
197
+
198
+ def _tokenize(self, value, index):
199
+ index = min(index, len(self._tokens) - 1)
200
+ return self._tokens[index]
201
+
202
+ def _is_assign(self, value):
203
+ if value.endswith('='):
204
+ value = value[:-1].strip()
205
+ var = VariableSplitter(value, identifiers='$@')
206
+ return var.start == 0 and var.end == len(value)
207
+
208
+
209
+ class Comment(Tokenizer):
210
+ _tokens = (COMMENT,)
211
+
212
+
213
+ class Setting(Tokenizer):
214
+ _tokens = (SETTING, ARGUMENT)
215
+ _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown',
216
+ 'suitepostcondition', 'testsetup', 'testprecondition',
217
+ 'testteardown', 'testpostcondition', 'testtemplate')
218
+ _import_settings = ('library', 'resource', 'variables')
219
+ _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags',
220
+ 'testtimeout')
221
+ _custom_tokenizer = None
222
+
223
+ def __init__(self, template_setter=None):
224
+ Tokenizer.__init__(self)
225
+ self._template_setter = template_setter
226
+
227
+ def _tokenize(self, value, index):
228
+ if index == 1 and self._template_setter:
229
+ self._template_setter(value)
230
+ if index == 0:
231
+ normalized = normalize(value)
232
+ if normalized in self._keyword_settings:
233
+ self._custom_tokenizer = KeywordCall(support_assign=False)
234
+ elif normalized in self._import_settings:
235
+ self._custom_tokenizer = ImportSetting()
236
+ elif normalized not in self._other_settings:
237
+ return ERROR
238
+ elif self._custom_tokenizer:
239
+ return self._custom_tokenizer.tokenize(value)
240
+ return Tokenizer._tokenize(self, value, index)
241
+
242
+
243
+ class ImportSetting(Tokenizer):
244
+ _tokens = (IMPORT, ARGUMENT)
245
+
246
+
247
+ class TestCaseSetting(Setting):
248
+ _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition',
249
+ 'template')
250
+ _import_settings = ()
251
+ _other_settings = ('documentation', 'tags', 'timeout')
252
+
253
+ def _tokenize(self, value, index):
254
+ if index == 0:
255
+ type = Setting._tokenize(self, value[1:-1], index)
256
+ return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)]
257
+ return Setting._tokenize(self, value, index)
258
+
259
+
260
+ class KeywordSetting(TestCaseSetting):
261
+ _keyword_settings = ('teardown',)
262
+ _other_settings = ('documentation', 'arguments', 'return', 'timeout')
263
+
264
+
265
+ class Variable(Tokenizer):
266
+ _tokens = (SYNTAX, ARGUMENT)
267
+
268
+ def _tokenize(self, value, index):
269
+ if index == 0 and not self._is_assign(value):
270
+ return ERROR
271
+ return Tokenizer._tokenize(self, value, index)
272
+
273
+
274
+ class KeywordCall(Tokenizer):
275
+ _tokens = (KEYWORD, ARGUMENT)
276
+
277
+ def __init__(self, support_assign=True):
278
+ Tokenizer.__init__(self)
279
+ self._keyword_found = not support_assign
280
+ self._assigns = 0
281
+
282
+ def _tokenize(self, value, index):
283
+ if not self._keyword_found and self._is_assign(value):
284
+ self._assigns += 1
285
+ return SYNTAX # VariableTokenizer tokenizes this later.
286
+ if self._keyword_found:
287
+ return Tokenizer._tokenize(self, value, index - self._assigns)
288
+ self._keyword_found = True
289
+ return GherkinTokenizer().tokenize(value, KEYWORD)
290
+
291
+
292
+ class GherkinTokenizer(object):
293
+ _gherkin_prefix = re.compile('^(Given|When|Then|And) ', re.IGNORECASE)
294
+
295
+ def tokenize(self, value, token):
296
+ match = self._gherkin_prefix.match(value)
297
+ if not match:
298
+ return [(value, token)]
299
+ end = match.end()
300
+ return [(value[:end], GHERKIN), (value[end:], token)]
301
+
302
+
303
+ class TemplatedKeywordCall(Tokenizer):
304
+ _tokens = (ARGUMENT,)
305
+
306
+
307
+ class ForLoop(Tokenizer):
308
+
309
+ def __init__(self):
310
+ Tokenizer.__init__(self)
311
+ self._in_arguments = False
312
+
313
+ def _tokenize(self, value, index):
314
+ token = self._in_arguments and ARGUMENT or SYNTAX
315
+ if value.upper() in ('IN', 'IN RANGE'):
316
+ self._in_arguments = True
317
+ return token
318
+
319
+
320
+ class _Table(object):
321
+ _tokenizer_class = None
322
+
323
+ def __init__(self, prev_tokenizer=None):
324
+ self._tokenizer = self._tokenizer_class()
325
+ self._prev_tokenizer = prev_tokenizer
326
+ self._prev_values_on_row = []
327
+
328
+ def tokenize(self, value, index):
329
+ if self._continues(value, index):
330
+ self._tokenizer = self._prev_tokenizer
331
+ yield value, SYNTAX
332
+ else:
333
+ for value_and_token in self._tokenize(value, index):
334
+ yield value_and_token
335
+ self._prev_values_on_row.append(value)
336
+
337
+ def _continues(self, value, index):
338
+ return value == '...' and all(self._is_empty(t)
339
+ for t in self._prev_values_on_row)
340
+
341
+ def _is_empty(self, value):
342
+ return value in ('', '\\')
343
+
344
+ def _tokenize(self, value, index):
345
+ return self._tokenizer.tokenize(value)
346
+
347
+ def end_row(self):
348
+ self.__init__(prev_tokenizer=self._tokenizer)
349
+
350
+
351
+ class UnknownTable(_Table):
352
+ _tokenizer_class = Comment
353
+
354
+ def _continues(self, value, index):
355
+ return False
356
+
357
+
358
+ class VariableTable(_Table):
359
+ _tokenizer_class = Variable
360
+
361
+
362
+ class SettingTable(_Table):
363
+ _tokenizer_class = Setting
364
+
365
+ def __init__(self, template_setter, prev_tokenizer=None):
366
+ _Table.__init__(self, prev_tokenizer)
367
+ self._template_setter = template_setter
368
+
369
+ def _tokenize(self, value, index):
370
+ if index == 0 and normalize(value) == 'testtemplate':
371
+ self._tokenizer = Setting(self._template_setter)
372
+ return _Table._tokenize(self, value, index)
373
+
374
+ def end_row(self):
375
+ self.__init__(self._template_setter, prev_tokenizer=self._tokenizer)
376
+
377
+
378
+ class TestCaseTable(_Table):
379
+ _setting_class = TestCaseSetting
380
+ _test_template = None
381
+ _default_template = None
382
+
383
+ @property
384
+ def _tokenizer_class(self):
385
+ if self._test_template or (self._default_template and
386
+ self._test_template is not False):
387
+ return TemplatedKeywordCall
388
+ return KeywordCall
389
+
390
+ def _continues(self, value, index):
391
+ return index > 0 and _Table._continues(self, value, index)
392
+
393
+ def _tokenize(self, value, index):
394
+ if index == 0:
395
+ if value:
396
+ self._test_template = None
397
+ return GherkinTokenizer().tokenize(value, TC_KW_NAME)
398
+ if index == 1 and self._is_setting(value):
399
+ if self._is_template(value):
400
+ self._test_template = False
401
+ self._tokenizer = self._setting_class(self.set_test_template)
402
+ else:
403
+ self._tokenizer = self._setting_class()
404
+ if index == 1 and self._is_for_loop(value):
405
+ self._tokenizer = ForLoop()
406
+ if index == 1 and self._is_empty(value):
407
+ return [(value, SYNTAX)]
408
+ return _Table._tokenize(self, value, index)
409
+
410
+ def _is_setting(self, value):
411
+ return value.startswith('[') and value.endswith(']')
412
+
413
+ def _is_template(self, value):
414
+ return normalize(value) == '[template]'
415
+
416
+ def _is_for_loop(self, value):
417
+ return value.startswith(':') and normalize(value, remove=':') == 'for'
418
+
419
+ def set_test_template(self, template):
420
+ self._test_template = self._is_template_set(template)
421
+
422
+ def set_default_template(self, template):
423
+ self._default_template = self._is_template_set(template)
424
+
425
+ def _is_template_set(self, template):
426
+ return normalize(template) not in ('', '\\', 'none', '${empty}')
427
+
428
+
429
+ class KeywordTable(TestCaseTable):
430
+ _tokenizer_class = KeywordCall
431
+ _setting_class = KeywordSetting
432
+
433
+ def _is_template(self, value):
434
+ return False
435
+
436
+
437
+ # Following code copied directly from Robot Framework 2.7.5.
438
+
439
+ class VariableSplitter:
440
+
441
+ def __init__(self, string, identifiers):
442
+ self.identifier = None
443
+ self.base = None
444
+ self.index = None
445
+ self.start = -1
446
+ self.end = -1
447
+ self._identifiers = identifiers
448
+ self._may_have_internal_variables = False
449
+ try:
450
+ self._split(string)
451
+ except ValueError:
452
+ pass
453
+ else:
454
+ self._finalize()
455
+
456
+ def get_replaced_base(self, variables):
457
+ if self._may_have_internal_variables:
458
+ return variables.replace_string(self.base)
459
+ return self.base
460
+
461
+ def _finalize(self):
462
+ self.identifier = self._variable_chars[0]
463
+ self.base = ''.join(self._variable_chars[2:-1])
464
+ self.end = self.start + len(self._variable_chars)
465
+ if self._has_list_variable_index():
466
+ self.index = ''.join(self._list_variable_index_chars[1:-1])
467
+ self.end += len(self._list_variable_index_chars)
468
+
469
+ def _has_list_variable_index(self):
470
+ return self._list_variable_index_chars\
471
+ and self._list_variable_index_chars[-1] == ']'
472
+
473
+ def _split(self, string):
474
+ start_index, max_index = self._find_variable(string)
475
+ self.start = start_index
476
+ self._open_curly = 1
477
+ self._state = self._variable_state
478
+ self._variable_chars = [string[start_index], '{']
479
+ self._list_variable_index_chars = []
480
+ self._string = string
481
+ start_index += 2
482
+ for index, char in enumerate(string[start_index:]):
483
+ index += start_index # Giving start to enumerate only in Py 2.6+
484
+ try:
485
+ self._state(char, index)
486
+ except StopIteration:
487
+ return
488
+ if index == max_index and not self._scanning_list_variable_index():
489
+ return
490
+
491
+ def _scanning_list_variable_index(self):
492
+ return self._state in [self._waiting_list_variable_index_state,
493
+ self._list_variable_index_state]
494
+
495
+ def _find_variable(self, string):
496
+ max_end_index = string.rfind('}')
497
+ if max_end_index == -1:
498
+ raise ValueError('No variable end found')
499
+ if self._is_escaped(string, max_end_index):
500
+ return self._find_variable(string[:max_end_index])
501
+ start_index = self._find_start_index(string, 1, max_end_index)
502
+ if start_index == -1:
503
+ raise ValueError('No variable start found')
504
+ return start_index, max_end_index
505
+
506
+ def _find_start_index(self, string, start, end):
507
+ index = string.find('{', start, end) - 1
508
+ if index < 0:
509
+ return -1
510
+ if self._start_index_is_ok(string, index):
511
+ return index
512
+ return self._find_start_index(string, index+2, end)
513
+
514
+ def _start_index_is_ok(self, string, index):
515
+ return string[index] in self._identifiers\
516
+ and not self._is_escaped(string, index)
517
+
518
+ def _is_escaped(self, string, index):
519
+ escaped = False
520
+ while index > 0 and string[index-1] == '\\':
521
+ index -= 1
522
+ escaped = not escaped
523
+ return escaped
524
+
525
+ def _variable_state(self, char, index):
526
+ self._variable_chars.append(char)
527
+ if char == '}' and not self._is_escaped(self._string, index):
528
+ self._open_curly -= 1
529
+ if self._open_curly == 0:
530
+ if not self._is_list_variable():
531
+ raise StopIteration
532
+ self._state = self._waiting_list_variable_index_state
533
+ elif char in self._identifiers:
534
+ self._state = self._internal_variable_start_state
535
+
536
+ def _is_list_variable(self):
537
+ return self._variable_chars[0] == '@'
538
+
539
+ def _internal_variable_start_state(self, char, index):
540
+ self._state = self._variable_state
541
+ if char == '{':
542
+ self._variable_chars.append(char)
543
+ self._open_curly += 1
544
+ self._may_have_internal_variables = True
545
+ else:
546
+ self._variable_state(char, index)
547
+
548
+ def _waiting_list_variable_index_state(self, char, index):
549
+ if char != '[':
550
+ raise StopIteration
551
+ self._list_variable_index_chars.append(char)
552
+ self._state = self._list_variable_index_state
553
+
554
+ def _list_variable_index_state(self, char, index):
555
+ self._list_variable_index_chars.append(char)
556
+ if char == ']':
557
+ raise StopIteration