pygments.rb 1.2.1 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (386) hide show
  1. checksums.yaml +5 -5
  2. data/.github/dependabot.yml +13 -0
  3. data/.github/workflows/ci.yml +40 -0
  4. data/.github/workflows/release.yml +24 -0
  5. data/.gitignore +4 -6
  6. data/.rubocop.yml +16 -0
  7. data/CHANGELOG.adoc +137 -0
  8. data/Gemfile +3 -1
  9. data/LICENSE +1 -1
  10. data/README.adoc +159 -0
  11. data/Rakefile +8 -32
  12. data/bench.rb +14 -8
  13. data/lib/pygments.rb +16 -14
  14. data/lib/pygments/lexer.rb +87 -65
  15. data/lib/pygments/mentos.py +23 -110
  16. data/lib/pygments/popen.rb +188 -284
  17. data/lib/pygments/version.rb +2 -2
  18. data/pygments.rb.gemspec +20 -13
  19. data/test/test_pygments.rb +50 -110
  20. data/vendor/pygments-main/{AUTHORS → Pygments-2.8.1.dist-info/AUTHORS} +26 -3
  21. data/vendor/pygments-main/Pygments-2.8.1.dist-info/INSTALLER +1 -0
  22. data/vendor/pygments-main/{LICENSE → Pygments-2.8.1.dist-info/LICENSE} +1 -1
  23. data/vendor/pygments-main/Pygments-2.8.1.dist-info/METADATA +48 -0
  24. data/vendor/pygments-main/Pygments-2.8.1.dist-info/RECORD +494 -0
  25. data/vendor/pygments-main/Pygments-2.8.1.dist-info/REQUESTED +0 -0
  26. data/vendor/pygments-main/Pygments-2.8.1.dist-info/WHEEL +5 -0
  27. data/vendor/pygments-main/Pygments-2.8.1.dist-info/entry_points.txt +3 -0
  28. data/vendor/pygments-main/Pygments-2.8.1.dist-info/top_level.txt +1 -0
  29. data/vendor/pygments-main/bin/pygmentize +8 -0
  30. data/vendor/pygments-main/pygments/__init__.py +6 -12
  31. data/vendor/pygments-main/pygments/__main__.py +17 -0
  32. data/vendor/pygments-main/pygments/cmdline.py +213 -184
  33. data/vendor/pygments-main/pygments/console.py +6 -10
  34. data/vendor/pygments-main/pygments/filter.py +4 -7
  35. data/vendor/pygments-main/pygments/filters/__init__.py +609 -22
  36. data/vendor/pygments-main/pygments/formatter.py +4 -5
  37. data/vendor/pygments-main/pygments/formatters/__init__.py +10 -10
  38. data/vendor/pygments-main/pygments/formatters/_mapping.py +1 -4
  39. data/vendor/pygments-main/pygments/formatters/bbcode.py +1 -2
  40. data/vendor/pygments-main/pygments/formatters/html.py +242 -139
  41. data/vendor/pygments-main/pygments/formatters/img.py +91 -50
  42. data/vendor/pygments-main/pygments/formatters/irc.py +39 -40
  43. data/vendor/pygments-main/pygments/formatters/latex.py +62 -33
  44. data/vendor/pygments-main/pygments/formatters/other.py +18 -17
  45. data/vendor/pygments-main/pygments/formatters/rtf.py +29 -30
  46. data/vendor/pygments-main/pygments/formatters/svg.py +40 -5
  47. data/vendor/pygments-main/pygments/formatters/terminal.py +25 -32
  48. data/vendor/pygments-main/pygments/formatters/terminal256.py +45 -14
  49. data/vendor/pygments-main/pygments/lexer.py +47 -44
  50. data/vendor/pygments-main/pygments/lexers/__init__.py +341 -0
  51. data/vendor/pygments-main/pygments/lexers/_asy_builtins.py +1644 -0
  52. data/vendor/pygments-main/pygments/lexers/_cl_builtins.py +231 -0
  53. data/vendor/pygments-main/pygments/lexers/_cocoa_builtins.py +75 -0
  54. data/vendor/pygments-main/pygments/lexers/_csound_builtins.py +1724 -0
  55. data/vendor/pygments-main/pygments/lexers/_lasso_builtins.py +5326 -0
  56. data/vendor/pygments-main/pygments/lexers/_lua_builtins.py +292 -0
  57. data/vendor/pygments-main/pygments/lexers/_mapping.py +554 -0
  58. data/vendor/pygments-main/pygments/lexers/_mql_builtins.py +1171 -0
  59. data/vendor/pygments-main/pygments/lexers/_mysql_builtins.py +1281 -0
  60. data/vendor/pygments-main/pygments/lexers/_openedge_builtins.py +2600 -0
  61. data/vendor/pygments-main/pygments/lexers/_php_builtins.py +4752 -0
  62. data/vendor/pygments-main/pygments/lexers/_postgres_builtins.py +677 -0
  63. data/vendor/pygments-main/pygments/lexers/_scilab_builtins.py +3093 -0
  64. data/vendor/pygments-main/pygments/lexers/_sourcemod_builtins.py +1160 -0
  65. data/vendor/pygments-main/pygments/lexers/_stan_builtins.py +557 -0
  66. data/vendor/pygments-main/pygments/lexers/_stata_builtins.py +457 -0
  67. data/vendor/pygments-main/pygments/lexers/_tsql_builtins.py +1003 -0
  68. data/vendor/pygments-main/pygments/lexers/_usd_builtins.py +112 -0
  69. data/vendor/pygments-main/pygments/lexers/_vbscript_builtins.py +279 -0
  70. data/vendor/pygments-main/pygments/lexers/_vim_builtins.py +1938 -0
  71. data/vendor/pygments-main/pygments/lexers/actionscript.py +244 -0
  72. data/vendor/pygments-main/pygments/lexers/agile.py +23 -0
  73. data/vendor/pygments-main/pygments/lexers/algebra.py +239 -0
  74. data/vendor/pygments-main/pygments/lexers/ambient.py +75 -0
  75. data/vendor/pygments-main/pygments/lexers/amdgpu.py +48 -0
  76. data/vendor/pygments-main/pygments/lexers/ampl.py +86 -0
  77. data/vendor/pygments-main/pygments/lexers/apl.py +100 -0
  78. data/vendor/pygments-main/pygments/lexers/archetype.py +317 -0
  79. data/vendor/pygments-main/pygments/lexers/arrow.py +116 -0
  80. data/vendor/pygments-main/pygments/lexers/asm.py +1004 -0
  81. data/vendor/pygments-main/pygments/lexers/automation.py +379 -0
  82. data/vendor/pygments-main/pygments/lexers/bare.py +103 -0
  83. data/vendor/pygments-main/pygments/lexers/basic.py +661 -0
  84. data/vendor/pygments-main/pygments/lexers/bibtex.py +159 -0
  85. data/vendor/pygments-main/pygments/lexers/boa.py +101 -0
  86. data/vendor/pygments-main/pygments/lexers/business.py +626 -0
  87. data/vendor/pygments-main/pygments/lexers/c_cpp.py +342 -0
  88. data/vendor/pygments-main/pygments/lexers/c_like.py +565 -0
  89. data/vendor/pygments-main/pygments/lexers/capnproto.py +77 -0
  90. data/vendor/pygments-main/pygments/lexers/cddl.py +190 -0
  91. data/vendor/pygments-main/pygments/lexers/chapel.py +111 -0
  92. data/vendor/pygments-main/pygments/lexers/clean.py +178 -0
  93. data/vendor/pygments-main/pygments/lexers/compiled.py +33 -0
  94. data/vendor/pygments-main/pygments/lexers/configs.py +985 -0
  95. data/vendor/pygments-main/pygments/lexers/console.py +113 -0
  96. data/vendor/pygments-main/pygments/lexers/crystal.py +366 -0
  97. data/vendor/pygments-main/pygments/lexers/csound.py +466 -0
  98. data/vendor/pygments-main/pygments/lexers/css.py +694 -0
  99. data/vendor/pygments-main/pygments/lexers/d.py +255 -0
  100. data/vendor/pygments-main/pygments/lexers/dalvik.py +124 -0
  101. data/vendor/pygments-main/pygments/lexers/data.py +697 -0
  102. data/vendor/pygments-main/pygments/lexers/devicetree.py +108 -0
  103. data/vendor/pygments-main/pygments/lexers/diff.py +164 -0
  104. data/vendor/pygments-main/pygments/lexers/dotnet.py +706 -0
  105. data/vendor/pygments-main/pygments/lexers/dsls.py +959 -0
  106. data/vendor/pygments-main/pygments/lexers/dylan.py +286 -0
  107. data/vendor/pygments-main/pygments/lexers/ecl.py +137 -0
  108. data/vendor/pygments-main/pygments/lexers/eiffel.py +64 -0
  109. data/vendor/pygments-main/pygments/lexers/elm.py +120 -0
  110. data/vendor/pygments-main/pygments/lexers/email.py +150 -0
  111. data/vendor/pygments-main/pygments/lexers/erlang.py +529 -0
  112. data/vendor/pygments-main/pygments/lexers/esoteric.py +303 -0
  113. data/vendor/pygments-main/pygments/lexers/ezhil.py +76 -0
  114. data/vendor/pygments-main/pygments/lexers/factor.py +343 -0
  115. data/vendor/pygments-main/pygments/lexers/fantom.py +249 -0
  116. data/vendor/pygments-main/pygments/lexers/felix.py +272 -0
  117. data/vendor/pygments-main/pygments/lexers/floscript.py +82 -0
  118. data/vendor/pygments-main/pygments/lexers/forth.py +177 -0
  119. data/vendor/pygments-main/pygments/lexers/fortran.py +208 -0
  120. data/vendor/pygments-main/pygments/lexers/foxpro.py +427 -0
  121. data/vendor/pygments-main/pygments/lexers/freefem.py +897 -0
  122. data/vendor/pygments-main/pygments/lexers/functional.py +20 -0
  123. data/vendor/pygments-main/pygments/lexers/futhark.py +110 -0
  124. data/vendor/pygments-main/pygments/lexers/gdscript.py +345 -0
  125. data/vendor/pygments-main/pygments/lexers/go.py +100 -0
  126. data/vendor/pygments-main/pygments/lexers/grammar_notation.py +269 -0
  127. data/vendor/pygments-main/pygments/lexers/graph.py +84 -0
  128. data/vendor/pygments-main/pygments/lexers/graphics.py +799 -0
  129. data/vendor/pygments-main/pygments/lexers/graphviz.py +57 -0
  130. data/vendor/pygments-main/pygments/lexers/haskell.py +869 -0
  131. data/vendor/pygments-main/pygments/lexers/haxe.py +935 -0
  132. data/vendor/pygments-main/pygments/lexers/hdl.py +468 -0
  133. data/vendor/pygments-main/pygments/lexers/hexdump.py +102 -0
  134. data/vendor/pygments-main/pygments/lexers/html.py +613 -0
  135. data/vendor/pygments-main/pygments/lexers/idl.py +280 -0
  136. data/vendor/pygments-main/pygments/lexers/igor.py +419 -0
  137. data/vendor/pygments-main/pygments/lexers/inferno.py +95 -0
  138. data/vendor/pygments-main/pygments/lexers/installers.py +321 -0
  139. data/vendor/pygments-main/pygments/lexers/int_fiction.py +1367 -0
  140. data/vendor/pygments-main/pygments/lexers/iolang.py +62 -0
  141. data/vendor/pygments-main/pygments/lexers/j.py +145 -0
  142. data/vendor/pygments-main/pygments/lexers/javascript.py +1536 -0
  143. data/vendor/pygments-main/pygments/lexers/julia.py +330 -0
  144. data/vendor/pygments-main/pygments/lexers/jvm.py +1739 -0
  145. data/vendor/pygments-main/pygments/lexers/lisp.py +2698 -0
  146. data/vendor/pygments-main/pygments/lexers/make.py +205 -0
  147. data/vendor/pygments-main/pygments/lexers/markup.py +762 -0
  148. data/vendor/pygments-main/pygments/lexers/math.py +20 -0
  149. data/vendor/pygments-main/pygments/lexers/matlab.py +3280 -0
  150. data/vendor/pygments-main/pygments/lexers/mime.py +225 -0
  151. data/vendor/pygments-main/pygments/lexers/ml.py +957 -0
  152. data/vendor/pygments-main/pygments/lexers/modeling.py +365 -0
  153. data/vendor/pygments-main/pygments/lexers/modula2.py +1579 -0
  154. data/vendor/pygments-main/pygments/lexers/monte.py +203 -0
  155. data/vendor/pygments-main/pygments/lexers/mosel.py +447 -0
  156. data/vendor/pygments-main/pygments/lexers/ncl.py +893 -0
  157. data/vendor/pygments-main/pygments/lexers/nimrod.py +158 -0
  158. data/vendor/pygments-main/pygments/lexers/nit.py +63 -0
  159. data/vendor/pygments-main/pygments/lexers/nix.py +135 -0
  160. data/vendor/pygments-main/pygments/lexers/oberon.py +120 -0
  161. data/vendor/pygments-main/pygments/lexers/objective.py +503 -0
  162. data/vendor/pygments-main/pygments/lexers/ooc.py +84 -0
  163. data/vendor/pygments-main/pygments/lexers/other.py +40 -0
  164. data/vendor/pygments-main/pygments/lexers/parasail.py +78 -0
  165. data/vendor/pygments-main/pygments/lexers/parsers.py +799 -0
  166. data/vendor/pygments-main/pygments/lexers/pascal.py +643 -0
  167. data/vendor/pygments-main/pygments/lexers/pawn.py +202 -0
  168. data/vendor/pygments-main/pygments/lexers/perl.py +731 -0
  169. data/vendor/pygments-main/pygments/lexers/php.py +320 -0
  170. data/vendor/pygments-main/pygments/lexers/pointless.py +70 -0
  171. data/vendor/pygments-main/pygments/lexers/pony.py +93 -0
  172. data/vendor/pygments-main/pygments/lexers/praat.py +301 -0
  173. data/vendor/pygments-main/pygments/lexers/prolog.py +305 -0
  174. data/vendor/pygments-main/pygments/lexers/promql.py +182 -0
  175. data/vendor/pygments-main/pygments/lexers/python.py +1158 -0
  176. data/vendor/pygments-main/pygments/lexers/qvt.py +151 -0
  177. data/vendor/pygments-main/pygments/lexers/r.py +190 -0
  178. data/vendor/pygments-main/pygments/lexers/rdf.py +462 -0
  179. data/vendor/pygments-main/pygments/lexers/rebol.py +430 -0
  180. data/vendor/pygments-main/pygments/lexers/resource.py +84 -0
  181. data/vendor/pygments-main/pygments/lexers/ride.py +138 -0
  182. data/vendor/pygments-main/pygments/lexers/rnc.py +66 -0
  183. data/vendor/pygments-main/pygments/lexers/roboconf.py +81 -0
  184. data/vendor/pygments-main/pygments/lexers/robotframework.py +551 -0
  185. data/vendor/pygments-main/pygments/lexers/ruby.py +523 -0
  186. data/vendor/pygments-main/pygments/lexers/rust.py +223 -0
  187. data/vendor/pygments-main/pygments/lexers/sas.py +227 -0
  188. data/vendor/pygments-main/pygments/lexers/scdoc.py +82 -0
  189. data/vendor/pygments-main/pygments/lexers/scripting.py +1283 -0
  190. data/vendor/pygments-main/pygments/lexers/sgf.py +60 -0
  191. data/vendor/pygments-main/pygments/lexers/shell.py +909 -0
  192. data/vendor/pygments-main/pygments/lexers/sieve.py +68 -0
  193. data/vendor/pygments-main/pygments/lexers/slash.py +184 -0
  194. data/vendor/pygments-main/pygments/lexers/smalltalk.py +194 -0
  195. data/vendor/pygments-main/pygments/lexers/smv.py +78 -0
  196. data/vendor/pygments-main/pygments/lexers/snobol.py +82 -0
  197. data/vendor/pygments-main/pygments/lexers/solidity.py +91 -0
  198. data/vendor/pygments-main/pygments/lexers/special.py +106 -0
  199. data/vendor/pygments-main/pygments/lexers/sql.py +832 -0
  200. data/vendor/pygments-main/pygments/lexers/stata.py +170 -0
  201. data/vendor/pygments-main/pygments/lexers/supercollider.py +94 -0
  202. data/vendor/pygments-main/pygments/lexers/tcl.py +144 -0
  203. data/vendor/pygments-main/pygments/lexers/templates.py +2263 -0
  204. data/vendor/pygments-main/pygments/lexers/teraterm.py +334 -0
  205. data/vendor/pygments-main/pygments/lexers/testing.py +206 -0
  206. data/vendor/pygments-main/pygments/lexers/text.py +25 -0
  207. data/vendor/pygments-main/pygments/lexers/textedit.py +168 -0
  208. data/vendor/pygments-main/pygments/lexers/textfmts.py +429 -0
  209. data/vendor/pygments-main/pygments/lexers/theorem.py +476 -0
  210. data/vendor/pygments-main/pygments/lexers/tnt.py +262 -0
  211. data/vendor/pygments-main/pygments/lexers/trafficscript.py +53 -0
  212. data/vendor/pygments-main/pygments/lexers/typoscript.py +218 -0
  213. data/vendor/pygments-main/pygments/lexers/unicon.py +411 -0
  214. data/vendor/pygments-main/pygments/lexers/urbi.py +145 -0
  215. data/vendor/pygments-main/pygments/lexers/usd.py +89 -0
  216. data/vendor/pygments-main/pygments/lexers/varnish.py +189 -0
  217. data/vendor/pygments-main/pygments/lexers/verification.py +113 -0
  218. data/vendor/pygments-main/pygments/lexers/web.py +23 -0
  219. data/vendor/pygments-main/pygments/lexers/webidl.py +298 -0
  220. data/vendor/pygments-main/pygments/lexers/webmisc.py +991 -0
  221. data/vendor/pygments-main/pygments/lexers/whiley.py +115 -0
  222. data/vendor/pygments-main/pygments/lexers/x10.py +68 -0
  223. data/vendor/pygments-main/pygments/lexers/xorg.py +36 -0
  224. data/vendor/pygments-main/pygments/lexers/yang.py +103 -0
  225. data/vendor/pygments-main/pygments/lexers/zig.py +123 -0
  226. data/vendor/pygments-main/pygments/modeline.py +1 -2
  227. data/vendor/pygments-main/pygments/plugin.py +4 -3
  228. data/vendor/pygments-main/pygments/regexopt.py +1 -2
  229. data/vendor/pygments-main/pygments/scanner.py +2 -3
  230. data/vendor/pygments-main/pygments/sphinxext.py +2 -5
  231. data/vendor/pygments-main/pygments/style.py +61 -25
  232. data/vendor/pygments-main/pygments/styles/__init__.py +12 -5
  233. data/vendor/pygments-main/pygments/styles/abap.py +1 -2
  234. data/vendor/pygments-main/pygments/styles/algol.py +1 -2
  235. data/vendor/pygments-main/pygments/styles/algol_nu.py +1 -2
  236. data/vendor/pygments-main/pygments/styles/arduino.py +2 -3
  237. data/vendor/pygments-main/pygments/styles/autumn.py +1 -2
  238. data/vendor/pygments-main/pygments/styles/borland.py +1 -2
  239. data/vendor/pygments-main/pygments/styles/bw.py +1 -2
  240. data/vendor/pygments-main/pygments/styles/colorful.py +1 -2
  241. data/vendor/pygments-main/pygments/styles/default.py +1 -2
  242. data/vendor/pygments-main/pygments/styles/emacs.py +1 -2
  243. data/vendor/pygments-main/pygments/styles/friendly.py +1 -2
  244. data/vendor/pygments-main/pygments/styles/fruity.py +1 -2
  245. data/vendor/pygments-main/pygments/styles/igor.py +1 -2
  246. data/vendor/pygments-main/pygments/styles/inkpot.py +66 -0
  247. data/vendor/pygments-main/pygments/styles/lovelace.py +1 -2
  248. data/vendor/pygments-main/pygments/styles/manni.py +1 -2
  249. data/vendor/pygments-main/pygments/styles/material.py +118 -0
  250. data/vendor/pygments-main/pygments/styles/monokai.py +4 -4
  251. data/vendor/pygments-main/pygments/styles/murphy.py +1 -2
  252. data/vendor/pygments-main/pygments/styles/native.py +1 -2
  253. data/vendor/pygments-main/pygments/styles/paraiso_dark.py +1 -2
  254. data/vendor/pygments-main/pygments/styles/paraiso_light.py +1 -2
  255. data/vendor/pygments-main/pygments/styles/pastie.py +1 -2
  256. data/vendor/pygments-main/pygments/styles/perldoc.py +1 -2
  257. data/vendor/pygments-main/pygments/styles/rainbow_dash.py +1 -2
  258. data/vendor/pygments-main/pygments/styles/rrt.py +1 -2
  259. data/vendor/pygments-main/pygments/styles/sas.py +1 -2
  260. data/vendor/pygments-main/pygments/styles/solarized.py +136 -0
  261. data/vendor/pygments-main/pygments/styles/stata_dark.py +40 -0
  262. data/vendor/pygments-main/pygments/styles/{stata.py → stata_light.py} +14 -16
  263. data/vendor/pygments-main/pygments/styles/tango.py +1 -2
  264. data/vendor/pygments-main/pygments/styles/trac.py +1 -2
  265. data/vendor/pygments-main/pygments/styles/vim.py +1 -2
  266. data/vendor/pygments-main/pygments/styles/vs.py +1 -2
  267. data/vendor/pygments-main/pygments/styles/xcode.py +1 -2
  268. data/vendor/pygments-main/pygments/styles/zenburn.py +80 -0
  269. data/vendor/pygments-main/pygments/token.py +1 -2
  270. data/vendor/pygments-main/pygments/unistring.py +47 -109
  271. data/vendor/pygments-main/pygments/util.py +16 -96
  272. metadata +81 -139
  273. data/CHANGELOG.md +0 -111
  274. data/README.md +0 -121
  275. data/cache-lexers.rb +0 -8
  276. data/circle.yml +0 -20
  277. data/test/test_data.c +0 -2581
  278. data/test/test_data.py +0 -514
  279. data/test/test_data_generated +0 -2582
  280. data/vendor/custom_lexers/github.py +0 -565
  281. data/vendor/pygments-main/CHANGES +0 -1186
  282. data/vendor/pygments-main/MANIFEST.in +0 -6
  283. data/vendor/pygments-main/Makefile +0 -65
  284. data/vendor/pygments-main/README.rst +0 -39
  285. data/vendor/pygments-main/REVISION +0 -1
  286. data/vendor/pygments-main/TODO +0 -12
  287. data/vendor/pygments-main/doc/Makefile +0 -153
  288. data/vendor/pygments-main/doc/_static/favicon.ico +0 -0
  289. data/vendor/pygments-main/doc/_static/logo_new.png +0 -0
  290. data/vendor/pygments-main/doc/_static/logo_only.png +0 -0
  291. data/vendor/pygments-main/doc/_templates/docssidebar.html +0 -3
  292. data/vendor/pygments-main/doc/_templates/indexsidebar.html +0 -25
  293. data/vendor/pygments-main/doc/_themes/pygments14/layout.html +0 -98
  294. data/vendor/pygments-main/doc/_themes/pygments14/static/bodybg.png +0 -0
  295. data/vendor/pygments-main/doc/_themes/pygments14/static/docbg.png +0 -0
  296. data/vendor/pygments-main/doc/_themes/pygments14/static/listitem.png +0 -0
  297. data/vendor/pygments-main/doc/_themes/pygments14/static/logo.png +0 -0
  298. data/vendor/pygments-main/doc/_themes/pygments14/static/pocoo.png +0 -0
  299. data/vendor/pygments-main/doc/_themes/pygments14/static/pygments14.css_t +0 -401
  300. data/vendor/pygments-main/doc/_themes/pygments14/theme.conf +0 -15
  301. data/vendor/pygments-main/doc/conf.py +0 -241
  302. data/vendor/pygments-main/doc/docs/api.rst +0 -354
  303. data/vendor/pygments-main/doc/docs/authors.rst +0 -4
  304. data/vendor/pygments-main/doc/docs/changelog.rst +0 -1
  305. data/vendor/pygments-main/doc/docs/cmdline.rst +0 -166
  306. data/vendor/pygments-main/doc/docs/filterdevelopment.rst +0 -71
  307. data/vendor/pygments-main/doc/docs/filters.rst +0 -41
  308. data/vendor/pygments-main/doc/docs/formatterdevelopment.rst +0 -169
  309. data/vendor/pygments-main/doc/docs/formatters.rst +0 -48
  310. data/vendor/pygments-main/doc/docs/index.rst +0 -66
  311. data/vendor/pygments-main/doc/docs/integrate.rst +0 -40
  312. data/vendor/pygments-main/doc/docs/java.rst +0 -70
  313. data/vendor/pygments-main/doc/docs/lexerdevelopment.rst +0 -728
  314. data/vendor/pygments-main/doc/docs/lexers.rst +0 -69
  315. data/vendor/pygments-main/doc/docs/moinmoin.rst +0 -39
  316. data/vendor/pygments-main/doc/docs/plugins.rst +0 -93
  317. data/vendor/pygments-main/doc/docs/quickstart.rst +0 -205
  318. data/vendor/pygments-main/doc/docs/rstdirective.rst +0 -22
  319. data/vendor/pygments-main/doc/docs/styles.rst +0 -201
  320. data/vendor/pygments-main/doc/docs/tokens.rst +0 -372
  321. data/vendor/pygments-main/doc/docs/unicode.rst +0 -58
  322. data/vendor/pygments-main/doc/download.rst +0 -41
  323. data/vendor/pygments-main/doc/faq.rst +0 -139
  324. data/vendor/pygments-main/doc/index.rst +0 -54
  325. data/vendor/pygments-main/doc/languages.rst +0 -154
  326. data/vendor/pygments-main/doc/make.bat +0 -190
  327. data/vendor/pygments-main/doc/pygmentize.1 +0 -94
  328. data/vendor/pygments-main/external/autopygmentize +0 -101
  329. data/vendor/pygments-main/external/lasso-builtins-generator-9.lasso +0 -162
  330. data/vendor/pygments-main/external/markdown-processor.py +0 -67
  331. data/vendor/pygments-main/external/moin-parser.py +0 -112
  332. data/vendor/pygments-main/external/pygments.bashcomp +0 -38
  333. data/vendor/pygments-main/external/rst-directive.py +0 -82
  334. data/vendor/pygments-main/pygmentize +0 -8
  335. data/vendor/pygments-main/requirements.txt +0 -5
  336. data/vendor/pygments-main/scripts/check_sources.py +0 -211
  337. data/vendor/pygments-main/scripts/debug_lexer.py +0 -246
  338. data/vendor/pygments-main/scripts/detect_missing_analyse_text.py +0 -33
  339. data/vendor/pygments-main/scripts/epydoc.css +0 -280
  340. data/vendor/pygments-main/scripts/get_vimkw.py +0 -74
  341. data/vendor/pygments-main/scripts/pylintrc +0 -301
  342. data/vendor/pygments-main/scripts/vim2pygments.py +0 -935
  343. data/vendor/pygments-main/setup.cfg +0 -10
  344. data/vendor/pygments-main/setup.py +0 -77
  345. data/vendor/pygments-main/tox.ini +0 -7
  346. data/vendor/simplejson/.gitignore +0 -10
  347. data/vendor/simplejson/.travis.yml +0 -5
  348. data/vendor/simplejson/CHANGES.txt +0 -291
  349. data/vendor/simplejson/LICENSE.txt +0 -19
  350. data/vendor/simplejson/MANIFEST.in +0 -5
  351. data/vendor/simplejson/README.rst +0 -19
  352. data/vendor/simplejson/conf.py +0 -179
  353. data/vendor/simplejson/index.rst +0 -628
  354. data/vendor/simplejson/scripts/make_docs.py +0 -18
  355. data/vendor/simplejson/setup.py +0 -104
  356. data/vendor/simplejson/simplejson/__init__.py +0 -510
  357. data/vendor/simplejson/simplejson/_speedups.c +0 -2745
  358. data/vendor/simplejson/simplejson/decoder.py +0 -425
  359. data/vendor/simplejson/simplejson/encoder.py +0 -567
  360. data/vendor/simplejson/simplejson/ordered_dict.py +0 -119
  361. data/vendor/simplejson/simplejson/scanner.py +0 -77
  362. data/vendor/simplejson/simplejson/tests/__init__.py +0 -67
  363. data/vendor/simplejson/simplejson/tests/test_bigint_as_string.py +0 -55
  364. data/vendor/simplejson/simplejson/tests/test_check_circular.py +0 -30
  365. data/vendor/simplejson/simplejson/tests/test_decimal.py +0 -66
  366. data/vendor/simplejson/simplejson/tests/test_decode.py +0 -83
  367. data/vendor/simplejson/simplejson/tests/test_default.py +0 -9
  368. data/vendor/simplejson/simplejson/tests/test_dump.py +0 -67
  369. data/vendor/simplejson/simplejson/tests/test_encode_basestring_ascii.py +0 -46
  370. data/vendor/simplejson/simplejson/tests/test_encode_for_html.py +0 -32
  371. data/vendor/simplejson/simplejson/tests/test_errors.py +0 -34
  372. data/vendor/simplejson/simplejson/tests/test_fail.py +0 -91
  373. data/vendor/simplejson/simplejson/tests/test_float.py +0 -19
  374. data/vendor/simplejson/simplejson/tests/test_indent.py +0 -86
  375. data/vendor/simplejson/simplejson/tests/test_item_sort_key.py +0 -20
  376. data/vendor/simplejson/simplejson/tests/test_namedtuple.py +0 -121
  377. data/vendor/simplejson/simplejson/tests/test_pass1.py +0 -76
  378. data/vendor/simplejson/simplejson/tests/test_pass2.py +0 -14
  379. data/vendor/simplejson/simplejson/tests/test_pass3.py +0 -20
  380. data/vendor/simplejson/simplejson/tests/test_recursion.py +0 -67
  381. data/vendor/simplejson/simplejson/tests/test_scanstring.py +0 -117
  382. data/vendor/simplejson/simplejson/tests/test_separators.py +0 -42
  383. data/vendor/simplejson/simplejson/tests/test_speedups.py +0 -20
  384. data/vendor/simplejson/simplejson/tests/test_tuple.py +0 -49
  385. data/vendor/simplejson/simplejson/tests/test_unicode.py +0 -109
  386. data/vendor/simplejson/simplejson/tool.py +0 -39
@@ -0,0 +1,138 @@
1
+ """
2
+ pygments.lexers.ride
3
+ ~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for the Ride programming language.
6
+
7
+ :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import RegexLexer, words, include
12
+ from pygments.token import Comment, Keyword, Name, Number, Punctuation, String, Text
13
+
14
+ __all__ = ['RideLexer']
15
+
16
+
17
+ class RideLexer(RegexLexer):
18
+ """
19
+ For `Ride <https://docs.wavesplatform.com/en/ride/about-ride.html>`_
20
+ source code.
21
+
22
+ .. versionadded:: 2.6
23
+ """
24
+
25
+ name = 'Ride'
26
+ aliases = ['ride']
27
+ filenames = ['*.ride']
28
+ mimetypes = ['text/x-ride']
29
+
30
+ validName = r'[a-zA-Z_][a-zA-Z0-9_\']*'
31
+
32
+ builtinOps = (
33
+ '||', '|', '>=', '>', '==', '!',
34
+ '=', '<=', '<', '::', ':+', ':', '!=', '/',
35
+ '.', '=>', '-', '+', '*', '&&', '%', '++',
36
+ )
37
+
38
+ globalVariablesName = (
39
+ 'NOALG', 'MD5', 'SHA1', 'SHA224', 'SHA256', 'SHA384', 'SHA512',
40
+ 'SHA3224', 'SHA3256', 'SHA3384', 'SHA3512', 'nil', 'this', 'unit',
41
+ 'height', 'lastBlock', 'Buy', 'Sell', 'CEILING', 'FLOOR', 'DOWN',
42
+ 'HALFDOWN', 'HALFEVEN', 'HALFUP', 'UP',
43
+ )
44
+
45
+ typesName = (
46
+ 'Unit', 'Int', 'Boolean', 'ByteVector', 'String', 'Address', 'Alias',
47
+ 'Transfer', 'AssetPair', 'DataEntry', 'Order', 'Transaction',
48
+ 'GenesisTransaction', 'PaymentTransaction', 'ReissueTransaction',
49
+ 'BurnTransaction', 'MassTransferTransaction', 'ExchangeTransaction',
50
+ 'TransferTransaction', 'SetAssetScriptTransaction',
51
+ 'InvokeScriptTransaction', 'IssueTransaction', 'LeaseTransaction',
52
+ 'LeaseCancelTransaction', 'CreateAliasTransaction',
53
+ 'SetScriptTransaction', 'SponsorFeeTransaction', 'DataTransaction',
54
+ 'WriteSet', 'AttachedPayment', 'ScriptTransfer', 'TransferSet',
55
+ 'ScriptResult', 'Invocation', 'Asset', 'BlockInfo', 'Issue', 'Reissue',
56
+ 'Burn', 'NoAlg', 'Md5', 'Sha1', 'Sha224', 'Sha256', 'Sha384', 'Sha512',
57
+ 'Sha3224', 'Sha3256', 'Sha3384', 'Sha3512', 'BinaryEntry',
58
+ 'BooleanEntry', 'IntegerEntry', 'StringEntry', 'List', 'Ceiling',
59
+ 'Down', 'Floor', 'HalfDown', 'HalfEven', 'HalfUp', 'Up',
60
+ )
61
+
62
+ functionsName = (
63
+ 'fraction', 'size', 'toBytes', 'take', 'drop', 'takeRight', 'dropRight',
64
+ 'toString', 'isDefined', 'extract', 'throw', 'getElement', 'value',
65
+ 'cons', 'toUtf8String', 'toInt', 'indexOf', 'lastIndexOf', 'split',
66
+ 'parseInt', 'parseIntValue', 'keccak256', 'blake2b256', 'sha256',
67
+ 'sigVerify', 'toBase58String', 'fromBase58String', 'toBase64String',
68
+ 'fromBase64String', 'transactionById', 'transactionHeightById',
69
+ 'getInteger', 'getBoolean', 'getBinary', 'getString',
70
+ 'addressFromPublicKey', 'addressFromString', 'addressFromRecipient',
71
+ 'assetBalance', 'wavesBalance', 'getIntegerValue', 'getBooleanValue',
72
+ 'getBinaryValue', 'getStringValue', 'addressFromStringValue',
73
+ 'assetInfo', 'rsaVerify', 'checkMerkleProof', 'median',
74
+ 'valueOrElse', 'valueOrErrorMessage', 'contains', 'log', 'pow',
75
+ 'toBase16String', 'fromBase16String', 'blockInfoByHeight',
76
+ 'transferTransactionById',
77
+ )
78
+
79
+ reservedWords = words((
80
+ 'match', 'case', 'else', 'func', 'if',
81
+ 'let', 'then', '@Callable', '@Verifier',
82
+ ), suffix=r'\b')
83
+
84
+ tokens = {
85
+ 'root': [
86
+ # Comments
87
+ (r'#.*', Comment.Single),
88
+ # Whitespace
89
+ (r'\s+', Text),
90
+ # Strings
91
+ (r'"', String, 'doublequote'),
92
+ (r'utf8\'', String, 'utf8quote'),
93
+ (r'base(58|64|16)\'', String, 'singlequote'),
94
+ # Keywords
95
+ (reservedWords, Keyword.Reserved),
96
+ (r'\{-#.*?#-\}', Keyword.Reserved),
97
+ (r'FOLD<\d+>', Keyword.Reserved),
98
+ # Types
99
+ (words(typesName), Keyword.Type),
100
+ # Main
101
+ # (specialName, Keyword.Reserved),
102
+ # Prefix Operators
103
+ (words(builtinOps, prefix=r'\(', suffix=r'\)'), Name.Function),
104
+ # Infix Operators
105
+ (words(builtinOps), Name.Function),
106
+ (words(globalVariablesName), Name.Function),
107
+ (words(functionsName), Name.Function),
108
+ # Numbers
109
+ include('numbers'),
110
+ # Variable Names
111
+ (validName, Name.Variable),
112
+ # Parens
113
+ (r'[,()\[\]{}]', Punctuation),
114
+ ],
115
+
116
+ 'doublequote': [
117
+ (r'\\u[0-9a-fA-F]{4}', String.Escape),
118
+ (r'\\[nrfvb\\"]', String.Escape),
119
+ (r'[^"]', String),
120
+ (r'"', String, '#pop'),
121
+ ],
122
+
123
+ 'utf8quote': [
124
+ (r'\\u[0-9a-fA-F]{4}', String.Escape),
125
+ (r'\\[nrfvb\\\']', String.Escape),
126
+ (r'[^\']', String),
127
+ (r'\'', String, '#pop'),
128
+ ],
129
+
130
+ 'singlequote': [
131
+ (r'[^\']', String),
132
+ (r'\'', String, '#pop'),
133
+ ],
134
+
135
+ 'numbers': [
136
+ (r'_?\d+', Number.Integer),
137
+ ],
138
+ }
@@ -0,0 +1,66 @@
1
+ """
2
+ pygments.lexers.rnc
3
+ ~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for Relax-NG Compact syntax
6
+
7
+ :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import RegexLexer
12
+ from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
13
+ Punctuation
14
+
15
+ __all__ = ['RNCCompactLexer']
16
+
17
+
18
+ class RNCCompactLexer(RegexLexer):
19
+ """
20
+ For `RelaxNG-compact <http://relaxng.org>`_ syntax.
21
+
22
+ .. versionadded:: 2.2
23
+ """
24
+
25
+ name = 'Relax-NG Compact'
26
+ aliases = ['rnc', 'rng-compact']
27
+ filenames = ['*.rnc']
28
+
29
+ tokens = {
30
+ 'root': [
31
+ (r'namespace\b', Keyword.Namespace),
32
+ (r'(?:default|datatypes)\b', Keyword.Declaration),
33
+ (r'##.*$', Comment.Preproc),
34
+ (r'#.*$', Comment.Single),
35
+ (r'"[^"]*"', String.Double),
36
+ # TODO single quoted strings and escape sequences outside of
37
+ # double-quoted strings
38
+ (r'(?:element|attribute|mixed)\b', Keyword.Declaration, 'variable'),
39
+ (r'(text\b|xsd:[^ ]+)', Keyword.Type, 'maybe_xsdattributes'),
40
+ (r'[,?&*=|~]|>>', Operator),
41
+ (r'[(){}]', Punctuation),
42
+ (r'.', Text),
43
+ ],
44
+
45
+ # a variable has been declared using `element` or `attribute`
46
+ 'variable': [
47
+ (r'[^{]+', Name.Variable),
48
+ (r'\{', Punctuation, '#pop'),
49
+ ],
50
+
51
+ # after an xsd:<datatype> declaration there may be attributes
52
+ 'maybe_xsdattributes': [
53
+ (r'\{', Punctuation, 'xsdattributes'),
54
+ (r'\}', Punctuation, '#pop'),
55
+ (r'.', Text),
56
+ ],
57
+
58
+ # attributes take the form { key1 = value1 key2 = value2 ... }
59
+ 'xsdattributes': [
60
+ (r'[^ =}]', Name.Attribute),
61
+ (r'=', Operator),
62
+ (r'"[^"]*"', String.Double),
63
+ (r'\}', Punctuation, '#pop'),
64
+ (r'.', Text),
65
+ ],
66
+ }
@@ -0,0 +1,81 @@
1
+ """
2
+ pygments.lexers.roboconf
3
+ ~~~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexers for Roboconf DSL.
6
+
7
+ :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import RegexLexer, words, re
12
+ from pygments.token import Text, Operator, Keyword, Name, Comment
13
+
14
+ __all__ = ['RoboconfGraphLexer', 'RoboconfInstancesLexer']
15
+
16
+
17
+ class RoboconfGraphLexer(RegexLexer):
18
+ """
19
+ Lexer for `Roboconf <http://roboconf.net/en/roboconf.html>`_ graph files.
20
+
21
+ .. versionadded:: 2.1
22
+ """
23
+ name = 'Roboconf Graph'
24
+ aliases = ['roboconf-graph']
25
+ filenames = ['*.graph']
26
+
27
+ flags = re.IGNORECASE | re.MULTILINE
28
+ tokens = {
29
+ 'root': [
30
+ # Skip white spaces
31
+ (r'\s+', Text),
32
+
33
+ # There is one operator
34
+ (r'=', Operator),
35
+
36
+ # Keywords
37
+ (words(('facet', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword),
38
+ (words((
39
+ 'installer', 'extends', 'exports', 'imports', 'facets',
40
+ 'children'), suffix=r'\s*:?', prefix=r'\b'), Name),
41
+
42
+ # Comments
43
+ (r'#.*\n', Comment),
44
+
45
+ # Default
46
+ (r'[^#]', Text),
47
+ (r'.*\n', Text)
48
+ ]
49
+ }
50
+
51
+
52
+ class RoboconfInstancesLexer(RegexLexer):
53
+ """
54
+ Lexer for `Roboconf <http://roboconf.net/en/roboconf.html>`_ instances files.
55
+
56
+ .. versionadded:: 2.1
57
+ """
58
+ name = 'Roboconf Instances'
59
+ aliases = ['roboconf-instances']
60
+ filenames = ['*.instances']
61
+
62
+ flags = re.IGNORECASE | re.MULTILINE
63
+ tokens = {
64
+ 'root': [
65
+
66
+ # Skip white spaces
67
+ (r'\s+', Text),
68
+
69
+ # Keywords
70
+ (words(('instance of', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword),
71
+ (words(('name', 'count'), suffix=r's*:?', prefix=r'\b'), Name),
72
+ (r'\s*[\w.-]+\s*:', Name),
73
+
74
+ # Comments
75
+ (r'#.*\n', Comment),
76
+
77
+ # Default
78
+ (r'[^#]', Text),
79
+ (r'.*\n', Text)
80
+ ]
81
+ }
@@ -0,0 +1,551 @@
1
+ """
2
+ pygments.lexers.robotframework
3
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for Robot Framework.
6
+
7
+ :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ # Copyright 2012 Nokia Siemens Networks Oyj
12
+ #
13
+ # Licensed under the Apache License, Version 2.0 (the "License");
14
+ # you may not use this file except in compliance with the License.
15
+ # You may obtain a copy of the License at
16
+ #
17
+ # http://www.apache.org/licenses/LICENSE-2.0
18
+ #
19
+ # Unless required by applicable law or agreed to in writing, software
20
+ # distributed under the License is distributed on an "AS IS" BASIS,
21
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22
+ # See the License for the specific language governing permissions and
23
+ # limitations under the License.
24
+
25
+ import re
26
+
27
+ from pygments.lexer import Lexer
28
+ from pygments.token import Token
29
+
30
+ __all__ = ['RobotFrameworkLexer']
31
+
32
+
33
+ HEADING = Token.Generic.Heading
34
+ SETTING = Token.Keyword.Namespace
35
+ IMPORT = Token.Name.Namespace
36
+ TC_KW_NAME = Token.Generic.Subheading
37
+ KEYWORD = Token.Name.Function
38
+ ARGUMENT = Token.String
39
+ VARIABLE = Token.Name.Variable
40
+ COMMENT = Token.Comment
41
+ SEPARATOR = Token.Punctuation
42
+ SYNTAX = Token.Punctuation
43
+ GHERKIN = Token.Generic.Emph
44
+ ERROR = Token.Error
45
+
46
+
47
+ def normalize(string, remove=''):
48
+ string = string.lower()
49
+ for char in remove + ' ':
50
+ if char in string:
51
+ string = string.replace(char, '')
52
+ return string
53
+
54
+
55
+ class RobotFrameworkLexer(Lexer):
56
+ """
57
+ For `Robot Framework <http://robotframework.org>`_ test data.
58
+
59
+ Supports both space and pipe separated plain text formats.
60
+
61
+ .. versionadded:: 1.6
62
+ """
63
+ name = 'RobotFramework'
64
+ aliases = ['robotframework']
65
+ filenames = ['*.robot']
66
+ mimetypes = ['text/x-robotframework']
67
+
68
+ def __init__(self, **options):
69
+ options['tabsize'] = 2
70
+ options['encoding'] = 'UTF-8'
71
+ Lexer.__init__(self, **options)
72
+
73
+ def get_tokens_unprocessed(self, text):
74
+ row_tokenizer = RowTokenizer()
75
+ var_tokenizer = VariableTokenizer()
76
+ index = 0
77
+ for row in text.splitlines():
78
+ for value, token in row_tokenizer.tokenize(row):
79
+ for value, token in var_tokenizer.tokenize(value, token):
80
+ if value:
81
+ yield index, token, str(value)
82
+ index += len(value)
83
+
84
+
85
+ class VariableTokenizer:
86
+
87
+ def tokenize(self, string, token):
88
+ var = VariableSplitter(string, identifiers='$@%&')
89
+ if var.start < 0 or token in (COMMENT, ERROR):
90
+ yield string, token
91
+ return
92
+ for value, token in self._tokenize(var, string, token):
93
+ if value:
94
+ yield value, token
95
+
96
+ def _tokenize(self, var, string, orig_token):
97
+ before = string[:var.start]
98
+ yield before, orig_token
99
+ yield var.identifier + '{', SYNTAX
100
+ yield from self.tokenize(var.base, VARIABLE)
101
+ yield '}', SYNTAX
102
+ if var.index:
103
+ yield '[', SYNTAX
104
+ yield from self.tokenize(var.index, VARIABLE)
105
+ yield ']', SYNTAX
106
+ yield from self.tokenize(string[var.end:], orig_token)
107
+
108
+
109
+ class RowTokenizer:
110
+
111
+ def __init__(self):
112
+ self._table = UnknownTable()
113
+ self._splitter = RowSplitter()
114
+ testcases = TestCaseTable()
115
+ settings = SettingTable(testcases.set_default_template)
116
+ variables = VariableTable()
117
+ keywords = KeywordTable()
118
+ self._tables = {'settings': settings, 'setting': settings,
119
+ 'metadata': settings,
120
+ 'variables': variables, 'variable': variables,
121
+ 'testcases': testcases, 'testcase': testcases,
122
+ 'tasks': testcases, 'task': testcases,
123
+ 'keywords': keywords, 'keyword': keywords,
124
+ 'userkeywords': keywords, 'userkeyword': keywords}
125
+
126
+ def tokenize(self, row):
127
+ commented = False
128
+ heading = False
129
+ for index, value in enumerate(self._splitter.split(row)):
130
+ # First value, and every second after that, is a separator.
131
+ index, separator = divmod(index-1, 2)
132
+ if value.startswith('#'):
133
+ commented = True
134
+ elif index == 0 and value.startswith('*'):
135
+ self._table = self._start_table(value)
136
+ heading = True
137
+ yield from self._tokenize(value, index, commented,
138
+ separator, heading)
139
+ self._table.end_row()
140
+
141
+ def _start_table(self, header):
142
+ name = normalize(header, remove='*')
143
+ return self._tables.get(name, UnknownTable())
144
+
145
+ def _tokenize(self, value, index, commented, separator, heading):
146
+ if commented:
147
+ yield value, COMMENT
148
+ elif separator:
149
+ yield value, SEPARATOR
150
+ elif heading:
151
+ yield value, HEADING
152
+ else:
153
+ yield from self._table.tokenize(value, index)
154
+
155
+
156
+ class RowSplitter:
157
+ _space_splitter = re.compile('( {2,})')
158
+ _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))')
159
+
160
+ def split(self, row):
161
+ splitter = (row.startswith('| ') and self._split_from_pipes
162
+ or self._split_from_spaces)
163
+ yield from splitter(row)
164
+ yield '\n'
165
+
166
+ def _split_from_spaces(self, row):
167
+ yield '' # Start with (pseudo)separator similarly as with pipes
168
+ yield from self._space_splitter.split(row)
169
+
170
+ def _split_from_pipes(self, row):
171
+ _, separator, rest = self._pipe_splitter.split(row, 1)
172
+ yield separator
173
+ while self._pipe_splitter.search(rest):
174
+ cell, separator, rest = self._pipe_splitter.split(rest, 1)
175
+ yield cell
176
+ yield separator
177
+ yield rest
178
+
179
+
180
+ class Tokenizer:
181
+ _tokens = None
182
+
183
+ def __init__(self):
184
+ self._index = 0
185
+
186
+ def tokenize(self, value):
187
+ values_and_tokens = self._tokenize(value, self._index)
188
+ self._index += 1
189
+ if isinstance(values_and_tokens, type(Token)):
190
+ values_and_tokens = [(value, values_and_tokens)]
191
+ return values_and_tokens
192
+
193
+ def _tokenize(self, value, index):
194
+ index = min(index, len(self._tokens) - 1)
195
+ return self._tokens[index]
196
+
197
+ def _is_assign(self, value):
198
+ if value.endswith('='):
199
+ value = value[:-1].strip()
200
+ var = VariableSplitter(value, identifiers='$@&')
201
+ return var.start == 0 and var.end == len(value)
202
+
203
+
204
+ class Comment(Tokenizer):
205
+ _tokens = (COMMENT,)
206
+
207
+
208
+ class Setting(Tokenizer):
209
+ _tokens = (SETTING, ARGUMENT)
210
+ _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown',
211
+ 'suitepostcondition', 'testsetup', 'tasksetup', 'testprecondition',
212
+ 'testteardown','taskteardown', 'testpostcondition', 'testtemplate', 'tasktemplate')
213
+ _import_settings = ('library', 'resource', 'variables')
214
+ _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags',
215
+ 'testtimeout','tasktimeout')
216
+ _custom_tokenizer = None
217
+
218
+ def __init__(self, template_setter=None):
219
+ Tokenizer.__init__(self)
220
+ self._template_setter = template_setter
221
+
222
+ def _tokenize(self, value, index):
223
+ if index == 1 and self._template_setter:
224
+ self._template_setter(value)
225
+ if index == 0:
226
+ normalized = normalize(value)
227
+ if normalized in self._keyword_settings:
228
+ self._custom_tokenizer = KeywordCall(support_assign=False)
229
+ elif normalized in self._import_settings:
230
+ self._custom_tokenizer = ImportSetting()
231
+ elif normalized not in self._other_settings:
232
+ return ERROR
233
+ elif self._custom_tokenizer:
234
+ return self._custom_tokenizer.tokenize(value)
235
+ return Tokenizer._tokenize(self, value, index)
236
+
237
+
238
+ class ImportSetting(Tokenizer):
239
+ _tokens = (IMPORT, ARGUMENT)
240
+
241
+
242
+ class TestCaseSetting(Setting):
243
+ _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition',
244
+ 'template')
245
+ _import_settings = ()
246
+ _other_settings = ('documentation', 'tags', 'timeout')
247
+
248
+ def _tokenize(self, value, index):
249
+ if index == 0:
250
+ type = Setting._tokenize(self, value[1:-1], index)
251
+ return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)]
252
+ return Setting._tokenize(self, value, index)
253
+
254
+
255
+ class KeywordSetting(TestCaseSetting):
256
+ _keyword_settings = ('teardown',)
257
+ _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags')
258
+
259
+
260
+ class Variable(Tokenizer):
261
+ _tokens = (SYNTAX, ARGUMENT)
262
+
263
+ def _tokenize(self, value, index):
264
+ if index == 0 and not self._is_assign(value):
265
+ return ERROR
266
+ return Tokenizer._tokenize(self, value, index)
267
+
268
+
269
+ class KeywordCall(Tokenizer):
270
+ _tokens = (KEYWORD, ARGUMENT)
271
+
272
+ def __init__(self, support_assign=True):
273
+ Tokenizer.__init__(self)
274
+ self._keyword_found = not support_assign
275
+ self._assigns = 0
276
+
277
+ def _tokenize(self, value, index):
278
+ if not self._keyword_found and self._is_assign(value):
279
+ self._assigns += 1
280
+ return SYNTAX # VariableTokenizer tokenizes this later.
281
+ if self._keyword_found:
282
+ return Tokenizer._tokenize(self, value, index - self._assigns)
283
+ self._keyword_found = True
284
+ return GherkinTokenizer().tokenize(value, KEYWORD)
285
+
286
+
287
+ class GherkinTokenizer:
288
+ _gherkin_prefix = re.compile('^(Given|When|Then|And) ', re.IGNORECASE)
289
+
290
+ def tokenize(self, value, token):
291
+ match = self._gherkin_prefix.match(value)
292
+ if not match:
293
+ return [(value, token)]
294
+ end = match.end()
295
+ return [(value[:end], GHERKIN), (value[end:], token)]
296
+
297
+
298
+ class TemplatedKeywordCall(Tokenizer):
299
+ _tokens = (ARGUMENT,)
300
+
301
+
302
+ class ForLoop(Tokenizer):
303
+
304
+ def __init__(self):
305
+ Tokenizer.__init__(self)
306
+ self._in_arguments = False
307
+
308
+ def _tokenize(self, value, index):
309
+ token = self._in_arguments and ARGUMENT or SYNTAX
310
+ if value.upper() in ('IN', 'IN RANGE'):
311
+ self._in_arguments = True
312
+ return token
313
+
314
+
315
+ class _Table:
316
+ _tokenizer_class = None
317
+
318
+ def __init__(self, prev_tokenizer=None):
319
+ self._tokenizer = self._tokenizer_class()
320
+ self._prev_tokenizer = prev_tokenizer
321
+ self._prev_values_on_row = []
322
+
323
+ def tokenize(self, value, index):
324
+ if self._continues(value, index):
325
+ self._tokenizer = self._prev_tokenizer
326
+ yield value, SYNTAX
327
+ else:
328
+ yield from self._tokenize(value, index)
329
+ self._prev_values_on_row.append(value)
330
+
331
+ def _continues(self, value, index):
332
+ return value == '...' and all(self._is_empty(t)
333
+ for t in self._prev_values_on_row)
334
+
335
+ def _is_empty(self, value):
336
+ return value in ('', '\\')
337
+
338
+ def _tokenize(self, value, index):
339
+ return self._tokenizer.tokenize(value)
340
+
341
+ def end_row(self):
342
+ self.__init__(prev_tokenizer=self._tokenizer)
343
+
344
+
345
+ class UnknownTable(_Table):
346
+ _tokenizer_class = Comment
347
+
348
+ def _continues(self, value, index):
349
+ return False
350
+
351
+
352
+ class VariableTable(_Table):
353
+ _tokenizer_class = Variable
354
+
355
+
356
+ class SettingTable(_Table):
357
+ _tokenizer_class = Setting
358
+
359
+ def __init__(self, template_setter, prev_tokenizer=None):
360
+ _Table.__init__(self, prev_tokenizer)
361
+ self._template_setter = template_setter
362
+
363
+ def _tokenize(self, value, index):
364
+ if index == 0 and normalize(value) == 'testtemplate':
365
+ self._tokenizer = Setting(self._template_setter)
366
+ return _Table._tokenize(self, value, index)
367
+
368
+ def end_row(self):
369
+ self.__init__(self._template_setter, prev_tokenizer=self._tokenizer)
370
+
371
+
372
+ class TestCaseTable(_Table):
373
+ _setting_class = TestCaseSetting
374
+ _test_template = None
375
+ _default_template = None
376
+
377
+ @property
378
+ def _tokenizer_class(self):
379
+ if self._test_template or (self._default_template and
380
+ self._test_template is not False):
381
+ return TemplatedKeywordCall
382
+ return KeywordCall
383
+
384
+ def _continues(self, value, index):
385
+ return index > 0 and _Table._continues(self, value, index)
386
+
387
+ def _tokenize(self, value, index):
388
+ if index == 0:
389
+ if value:
390
+ self._test_template = None
391
+ return GherkinTokenizer().tokenize(value, TC_KW_NAME)
392
+ if index == 1 and self._is_setting(value):
393
+ if self._is_template(value):
394
+ self._test_template = False
395
+ self._tokenizer = self._setting_class(self.set_test_template)
396
+ else:
397
+ self._tokenizer = self._setting_class()
398
+ if index == 1 and self._is_for_loop(value):
399
+ self._tokenizer = ForLoop()
400
+ if index == 1 and self._is_empty(value):
401
+ return [(value, SYNTAX)]
402
+ return _Table._tokenize(self, value, index)
403
+
404
+ def _is_setting(self, value):
405
+ return value.startswith('[') and value.endswith(']')
406
+
407
+ def _is_template(self, value):
408
+ return normalize(value) == '[template]'
409
+
410
+ def _is_for_loop(self, value):
411
+ return value.startswith(':') and normalize(value, remove=':') == 'for'
412
+
413
+ def set_test_template(self, template):
414
+ self._test_template = self._is_template_set(template)
415
+
416
+ def set_default_template(self, template):
417
+ self._default_template = self._is_template_set(template)
418
+
419
+ def _is_template_set(self, template):
420
+ return normalize(template) not in ('', '\\', 'none', '${empty}')
421
+
422
+
423
+ class KeywordTable(TestCaseTable):
424
+ _tokenizer_class = KeywordCall
425
+ _setting_class = KeywordSetting
426
+
427
+ def _is_template(self, value):
428
+ return False
429
+
430
+
431
+ # Following code copied directly from Robot Framework 2.7.5.
432
+
433
+ class VariableSplitter:
434
+
435
+ def __init__(self, string, identifiers):
436
+ self.identifier = None
437
+ self.base = None
438
+ self.index = None
439
+ self.start = -1
440
+ self.end = -1
441
+ self._identifiers = identifiers
442
+ self._may_have_internal_variables = False
443
+ try:
444
+ self._split(string)
445
+ except ValueError:
446
+ pass
447
+ else:
448
+ self._finalize()
449
+
450
+ def get_replaced_base(self, variables):
451
+ if self._may_have_internal_variables:
452
+ return variables.replace_string(self.base)
453
+ return self.base
454
+
455
+ def _finalize(self):
456
+ self.identifier = self._variable_chars[0]
457
+ self.base = ''.join(self._variable_chars[2:-1])
458
+ self.end = self.start + len(self._variable_chars)
459
+ if self._has_list_or_dict_variable_index():
460
+ self.index = ''.join(self._list_and_dict_variable_index_chars[1:-1])
461
+ self.end += len(self._list_and_dict_variable_index_chars)
462
+
463
+ def _has_list_or_dict_variable_index(self):
464
+ return self._list_and_dict_variable_index_chars\
465
+ and self._list_and_dict_variable_index_chars[-1] == ']'
466
+
467
+ def _split(self, string):
468
+ start_index, max_index = self._find_variable(string)
469
+ self.start = start_index
470
+ self._open_curly = 1
471
+ self._state = self._variable_state
472
+ self._variable_chars = [string[start_index], '{']
473
+ self._list_and_dict_variable_index_chars = []
474
+ self._string = string
475
+ start_index += 2
476
+ for index, char in enumerate(string[start_index:]):
477
+ index += start_index # Giving start to enumerate only in Py 2.6+
478
+ try:
479
+ self._state(char, index)
480
+ except StopIteration:
481
+ return
482
+ if index == max_index and not self._scanning_list_variable_index():
483
+ return
484
+
485
+ def _scanning_list_variable_index(self):
486
+ return self._state in [self._waiting_list_variable_index_state,
487
+ self._list_variable_index_state]
488
+
489
+ def _find_variable(self, string):
490
+ max_end_index = string.rfind('}')
491
+ if max_end_index == -1:
492
+ raise ValueError('No variable end found')
493
+ if self._is_escaped(string, max_end_index):
494
+ return self._find_variable(string[:max_end_index])
495
+ start_index = self._find_start_index(string, 1, max_end_index)
496
+ if start_index == -1:
497
+ raise ValueError('No variable start found')
498
+ return start_index, max_end_index
499
+
500
+ def _find_start_index(self, string, start, end):
501
+ index = string.find('{', start, end) - 1
502
+ if index < 0:
503
+ return -1
504
+ if self._start_index_is_ok(string, index):
505
+ return index
506
+ return self._find_start_index(string, index+2, end)
507
+
508
+ def _start_index_is_ok(self, string, index):
509
+ return string[index] in self._identifiers\
510
+ and not self._is_escaped(string, index)
511
+
512
+ def _is_escaped(self, string, index):
513
+ escaped = False
514
+ while index > 0 and string[index-1] == '\\':
515
+ index -= 1
516
+ escaped = not escaped
517
+ return escaped
518
+
519
+ def _variable_state(self, char, index):
520
+ self._variable_chars.append(char)
521
+ if char == '}' and not self._is_escaped(self._string, index):
522
+ self._open_curly -= 1
523
+ if self._open_curly == 0:
524
+ if not self._is_list_or_dict_variable():
525
+ raise StopIteration
526
+ self._state = self._waiting_list_variable_index_state
527
+ elif char in self._identifiers:
528
+ self._state = self._internal_variable_start_state
529
+
530
+ def _is_list_or_dict_variable(self):
531
+ return self._variable_chars[0] in ('@','&')
532
+
533
+ def _internal_variable_start_state(self, char, index):
534
+ self._state = self._variable_state
535
+ if char == '{':
536
+ self._variable_chars.append(char)
537
+ self._open_curly += 1
538
+ self._may_have_internal_variables = True
539
+ else:
540
+ self._variable_state(char, index)
541
+
542
+ def _waiting_list_variable_index_state(self, char, index):
543
+ if char != '[':
544
+ raise StopIteration
545
+ self._list_and_dict_variable_index_chars.append(char)
546
+ self._state = self._list_variable_index_state
547
+
548
+ def _list_variable_index_state(self, char, index):
549
+ self._list_and_dict_variable_index_chars.append(char)
550
+ if char == ']':
551
+ raise StopIteration