pygments.rb 1.2.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (386) hide show
  1. checksums.yaml +5 -5
  2. data/.github/dependabot.yml +13 -0
  3. data/.github/workflows/ci.yml +40 -0
  4. data/.github/workflows/release.yml +24 -0
  5. data/.gitignore +4 -6
  6. data/.rubocop.yml +16 -0
  7. data/CHANGELOG.adoc +137 -0
  8. data/Gemfile +3 -1
  9. data/LICENSE +1 -1
  10. data/README.adoc +159 -0
  11. data/Rakefile +8 -32
  12. data/bench.rb +14 -8
  13. data/lib/pygments.rb +16 -14
  14. data/lib/pygments/lexer.rb +87 -65
  15. data/lib/pygments/mentos.py +23 -110
  16. data/lib/pygments/popen.rb +188 -284
  17. data/lib/pygments/version.rb +2 -2
  18. data/pygments.rb.gemspec +20 -13
  19. data/test/test_pygments.rb +50 -110
  20. data/vendor/pygments-main/{AUTHORS → Pygments-2.8.1.dist-info/AUTHORS} +26 -3
  21. data/vendor/pygments-main/Pygments-2.8.1.dist-info/INSTALLER +1 -0
  22. data/vendor/pygments-main/{LICENSE → Pygments-2.8.1.dist-info/LICENSE} +1 -1
  23. data/vendor/pygments-main/Pygments-2.8.1.dist-info/METADATA +48 -0
  24. data/vendor/pygments-main/Pygments-2.8.1.dist-info/RECORD +494 -0
  25. data/vendor/pygments-main/Pygments-2.8.1.dist-info/REQUESTED +0 -0
  26. data/vendor/pygments-main/Pygments-2.8.1.dist-info/WHEEL +5 -0
  27. data/vendor/pygments-main/Pygments-2.8.1.dist-info/entry_points.txt +3 -0
  28. data/vendor/pygments-main/Pygments-2.8.1.dist-info/top_level.txt +1 -0
  29. data/vendor/pygments-main/bin/pygmentize +8 -0
  30. data/vendor/pygments-main/pygments/__init__.py +6 -12
  31. data/vendor/pygments-main/pygments/__main__.py +17 -0
  32. data/vendor/pygments-main/pygments/cmdline.py +213 -184
  33. data/vendor/pygments-main/pygments/console.py +6 -10
  34. data/vendor/pygments-main/pygments/filter.py +4 -7
  35. data/vendor/pygments-main/pygments/filters/__init__.py +609 -22
  36. data/vendor/pygments-main/pygments/formatter.py +4 -5
  37. data/vendor/pygments-main/pygments/formatters/__init__.py +10 -10
  38. data/vendor/pygments-main/pygments/formatters/_mapping.py +1 -4
  39. data/vendor/pygments-main/pygments/formatters/bbcode.py +1 -2
  40. data/vendor/pygments-main/pygments/formatters/html.py +242 -139
  41. data/vendor/pygments-main/pygments/formatters/img.py +91 -50
  42. data/vendor/pygments-main/pygments/formatters/irc.py +39 -40
  43. data/vendor/pygments-main/pygments/formatters/latex.py +62 -33
  44. data/vendor/pygments-main/pygments/formatters/other.py +18 -17
  45. data/vendor/pygments-main/pygments/formatters/rtf.py +29 -30
  46. data/vendor/pygments-main/pygments/formatters/svg.py +40 -5
  47. data/vendor/pygments-main/pygments/formatters/terminal.py +25 -32
  48. data/vendor/pygments-main/pygments/formatters/terminal256.py +45 -14
  49. data/vendor/pygments-main/pygments/lexer.py +47 -44
  50. data/vendor/pygments-main/pygments/lexers/__init__.py +341 -0
  51. data/vendor/pygments-main/pygments/lexers/_asy_builtins.py +1644 -0
  52. data/vendor/pygments-main/pygments/lexers/_cl_builtins.py +231 -0
  53. data/vendor/pygments-main/pygments/lexers/_cocoa_builtins.py +75 -0
  54. data/vendor/pygments-main/pygments/lexers/_csound_builtins.py +1724 -0
  55. data/vendor/pygments-main/pygments/lexers/_lasso_builtins.py +5326 -0
  56. data/vendor/pygments-main/pygments/lexers/_lua_builtins.py +292 -0
  57. data/vendor/pygments-main/pygments/lexers/_mapping.py +554 -0
  58. data/vendor/pygments-main/pygments/lexers/_mql_builtins.py +1171 -0
  59. data/vendor/pygments-main/pygments/lexers/_mysql_builtins.py +1281 -0
  60. data/vendor/pygments-main/pygments/lexers/_openedge_builtins.py +2600 -0
  61. data/vendor/pygments-main/pygments/lexers/_php_builtins.py +4752 -0
  62. data/vendor/pygments-main/pygments/lexers/_postgres_builtins.py +677 -0
  63. data/vendor/pygments-main/pygments/lexers/_scilab_builtins.py +3093 -0
  64. data/vendor/pygments-main/pygments/lexers/_sourcemod_builtins.py +1160 -0
  65. data/vendor/pygments-main/pygments/lexers/_stan_builtins.py +557 -0
  66. data/vendor/pygments-main/pygments/lexers/_stata_builtins.py +457 -0
  67. data/vendor/pygments-main/pygments/lexers/_tsql_builtins.py +1003 -0
  68. data/vendor/pygments-main/pygments/lexers/_usd_builtins.py +112 -0
  69. data/vendor/pygments-main/pygments/lexers/_vbscript_builtins.py +279 -0
  70. data/vendor/pygments-main/pygments/lexers/_vim_builtins.py +1938 -0
  71. data/vendor/pygments-main/pygments/lexers/actionscript.py +244 -0
  72. data/vendor/pygments-main/pygments/lexers/agile.py +23 -0
  73. data/vendor/pygments-main/pygments/lexers/algebra.py +239 -0
  74. data/vendor/pygments-main/pygments/lexers/ambient.py +75 -0
  75. data/vendor/pygments-main/pygments/lexers/amdgpu.py +48 -0
  76. data/vendor/pygments-main/pygments/lexers/ampl.py +86 -0
  77. data/vendor/pygments-main/pygments/lexers/apl.py +100 -0
  78. data/vendor/pygments-main/pygments/lexers/archetype.py +317 -0
  79. data/vendor/pygments-main/pygments/lexers/arrow.py +116 -0
  80. data/vendor/pygments-main/pygments/lexers/asm.py +1004 -0
  81. data/vendor/pygments-main/pygments/lexers/automation.py +379 -0
  82. data/vendor/pygments-main/pygments/lexers/bare.py +103 -0
  83. data/vendor/pygments-main/pygments/lexers/basic.py +661 -0
  84. data/vendor/pygments-main/pygments/lexers/bibtex.py +159 -0
  85. data/vendor/pygments-main/pygments/lexers/boa.py +101 -0
  86. data/vendor/pygments-main/pygments/lexers/business.py +626 -0
  87. data/vendor/pygments-main/pygments/lexers/c_cpp.py +342 -0
  88. data/vendor/pygments-main/pygments/lexers/c_like.py +565 -0
  89. data/vendor/pygments-main/pygments/lexers/capnproto.py +77 -0
  90. data/vendor/pygments-main/pygments/lexers/cddl.py +190 -0
  91. data/vendor/pygments-main/pygments/lexers/chapel.py +111 -0
  92. data/vendor/pygments-main/pygments/lexers/clean.py +178 -0
  93. data/vendor/pygments-main/pygments/lexers/compiled.py +33 -0
  94. data/vendor/pygments-main/pygments/lexers/configs.py +985 -0
  95. data/vendor/pygments-main/pygments/lexers/console.py +113 -0
  96. data/vendor/pygments-main/pygments/lexers/crystal.py +366 -0
  97. data/vendor/pygments-main/pygments/lexers/csound.py +466 -0
  98. data/vendor/pygments-main/pygments/lexers/css.py +694 -0
  99. data/vendor/pygments-main/pygments/lexers/d.py +255 -0
  100. data/vendor/pygments-main/pygments/lexers/dalvik.py +124 -0
  101. data/vendor/pygments-main/pygments/lexers/data.py +697 -0
  102. data/vendor/pygments-main/pygments/lexers/devicetree.py +108 -0
  103. data/vendor/pygments-main/pygments/lexers/diff.py +164 -0
  104. data/vendor/pygments-main/pygments/lexers/dotnet.py +706 -0
  105. data/vendor/pygments-main/pygments/lexers/dsls.py +959 -0
  106. data/vendor/pygments-main/pygments/lexers/dylan.py +286 -0
  107. data/vendor/pygments-main/pygments/lexers/ecl.py +137 -0
  108. data/vendor/pygments-main/pygments/lexers/eiffel.py +64 -0
  109. data/vendor/pygments-main/pygments/lexers/elm.py +120 -0
  110. data/vendor/pygments-main/pygments/lexers/email.py +150 -0
  111. data/vendor/pygments-main/pygments/lexers/erlang.py +529 -0
  112. data/vendor/pygments-main/pygments/lexers/esoteric.py +303 -0
  113. data/vendor/pygments-main/pygments/lexers/ezhil.py +76 -0
  114. data/vendor/pygments-main/pygments/lexers/factor.py +343 -0
  115. data/vendor/pygments-main/pygments/lexers/fantom.py +249 -0
  116. data/vendor/pygments-main/pygments/lexers/felix.py +272 -0
  117. data/vendor/pygments-main/pygments/lexers/floscript.py +82 -0
  118. data/vendor/pygments-main/pygments/lexers/forth.py +177 -0
  119. data/vendor/pygments-main/pygments/lexers/fortran.py +208 -0
  120. data/vendor/pygments-main/pygments/lexers/foxpro.py +427 -0
  121. data/vendor/pygments-main/pygments/lexers/freefem.py +897 -0
  122. data/vendor/pygments-main/pygments/lexers/functional.py +20 -0
  123. data/vendor/pygments-main/pygments/lexers/futhark.py +110 -0
  124. data/vendor/pygments-main/pygments/lexers/gdscript.py +345 -0
  125. data/vendor/pygments-main/pygments/lexers/go.py +100 -0
  126. data/vendor/pygments-main/pygments/lexers/grammar_notation.py +269 -0
  127. data/vendor/pygments-main/pygments/lexers/graph.py +84 -0
  128. data/vendor/pygments-main/pygments/lexers/graphics.py +799 -0
  129. data/vendor/pygments-main/pygments/lexers/graphviz.py +57 -0
  130. data/vendor/pygments-main/pygments/lexers/haskell.py +869 -0
  131. data/vendor/pygments-main/pygments/lexers/haxe.py +935 -0
  132. data/vendor/pygments-main/pygments/lexers/hdl.py +468 -0
  133. data/vendor/pygments-main/pygments/lexers/hexdump.py +102 -0
  134. data/vendor/pygments-main/pygments/lexers/html.py +613 -0
  135. data/vendor/pygments-main/pygments/lexers/idl.py +280 -0
  136. data/vendor/pygments-main/pygments/lexers/igor.py +419 -0
  137. data/vendor/pygments-main/pygments/lexers/inferno.py +95 -0
  138. data/vendor/pygments-main/pygments/lexers/installers.py +321 -0
  139. data/vendor/pygments-main/pygments/lexers/int_fiction.py +1367 -0
  140. data/vendor/pygments-main/pygments/lexers/iolang.py +62 -0
  141. data/vendor/pygments-main/pygments/lexers/j.py +145 -0
  142. data/vendor/pygments-main/pygments/lexers/javascript.py +1536 -0
  143. data/vendor/pygments-main/pygments/lexers/julia.py +330 -0
  144. data/vendor/pygments-main/pygments/lexers/jvm.py +1739 -0
  145. data/vendor/pygments-main/pygments/lexers/lisp.py +2698 -0
  146. data/vendor/pygments-main/pygments/lexers/make.py +205 -0
  147. data/vendor/pygments-main/pygments/lexers/markup.py +762 -0
  148. data/vendor/pygments-main/pygments/lexers/math.py +20 -0
  149. data/vendor/pygments-main/pygments/lexers/matlab.py +3280 -0
  150. data/vendor/pygments-main/pygments/lexers/mime.py +225 -0
  151. data/vendor/pygments-main/pygments/lexers/ml.py +957 -0
  152. data/vendor/pygments-main/pygments/lexers/modeling.py +365 -0
  153. data/vendor/pygments-main/pygments/lexers/modula2.py +1579 -0
  154. data/vendor/pygments-main/pygments/lexers/monte.py +203 -0
  155. data/vendor/pygments-main/pygments/lexers/mosel.py +447 -0
  156. data/vendor/pygments-main/pygments/lexers/ncl.py +893 -0
  157. data/vendor/pygments-main/pygments/lexers/nimrod.py +158 -0
  158. data/vendor/pygments-main/pygments/lexers/nit.py +63 -0
  159. data/vendor/pygments-main/pygments/lexers/nix.py +135 -0
  160. data/vendor/pygments-main/pygments/lexers/oberon.py +120 -0
  161. data/vendor/pygments-main/pygments/lexers/objective.py +503 -0
  162. data/vendor/pygments-main/pygments/lexers/ooc.py +84 -0
  163. data/vendor/pygments-main/pygments/lexers/other.py +40 -0
  164. data/vendor/pygments-main/pygments/lexers/parasail.py +78 -0
  165. data/vendor/pygments-main/pygments/lexers/parsers.py +799 -0
  166. data/vendor/pygments-main/pygments/lexers/pascal.py +643 -0
  167. data/vendor/pygments-main/pygments/lexers/pawn.py +202 -0
  168. data/vendor/pygments-main/pygments/lexers/perl.py +731 -0
  169. data/vendor/pygments-main/pygments/lexers/php.py +320 -0
  170. data/vendor/pygments-main/pygments/lexers/pointless.py +70 -0
  171. data/vendor/pygments-main/pygments/lexers/pony.py +93 -0
  172. data/vendor/pygments-main/pygments/lexers/praat.py +301 -0
  173. data/vendor/pygments-main/pygments/lexers/prolog.py +305 -0
  174. data/vendor/pygments-main/pygments/lexers/promql.py +182 -0
  175. data/vendor/pygments-main/pygments/lexers/python.py +1158 -0
  176. data/vendor/pygments-main/pygments/lexers/qvt.py +151 -0
  177. data/vendor/pygments-main/pygments/lexers/r.py +190 -0
  178. data/vendor/pygments-main/pygments/lexers/rdf.py +462 -0
  179. data/vendor/pygments-main/pygments/lexers/rebol.py +430 -0
  180. data/vendor/pygments-main/pygments/lexers/resource.py +84 -0
  181. data/vendor/pygments-main/pygments/lexers/ride.py +138 -0
  182. data/vendor/pygments-main/pygments/lexers/rnc.py +66 -0
  183. data/vendor/pygments-main/pygments/lexers/roboconf.py +81 -0
  184. data/vendor/pygments-main/pygments/lexers/robotframework.py +551 -0
  185. data/vendor/pygments-main/pygments/lexers/ruby.py +523 -0
  186. data/vendor/pygments-main/pygments/lexers/rust.py +223 -0
  187. data/vendor/pygments-main/pygments/lexers/sas.py +227 -0
  188. data/vendor/pygments-main/pygments/lexers/scdoc.py +82 -0
  189. data/vendor/pygments-main/pygments/lexers/scripting.py +1283 -0
  190. data/vendor/pygments-main/pygments/lexers/sgf.py +60 -0
  191. data/vendor/pygments-main/pygments/lexers/shell.py +909 -0
  192. data/vendor/pygments-main/pygments/lexers/sieve.py +68 -0
  193. data/vendor/pygments-main/pygments/lexers/slash.py +184 -0
  194. data/vendor/pygments-main/pygments/lexers/smalltalk.py +194 -0
  195. data/vendor/pygments-main/pygments/lexers/smv.py +78 -0
  196. data/vendor/pygments-main/pygments/lexers/snobol.py +82 -0
  197. data/vendor/pygments-main/pygments/lexers/solidity.py +91 -0
  198. data/vendor/pygments-main/pygments/lexers/special.py +106 -0
  199. data/vendor/pygments-main/pygments/lexers/sql.py +832 -0
  200. data/vendor/pygments-main/pygments/lexers/stata.py +170 -0
  201. data/vendor/pygments-main/pygments/lexers/supercollider.py +94 -0
  202. data/vendor/pygments-main/pygments/lexers/tcl.py +144 -0
  203. data/vendor/pygments-main/pygments/lexers/templates.py +2263 -0
  204. data/vendor/pygments-main/pygments/lexers/teraterm.py +334 -0
  205. data/vendor/pygments-main/pygments/lexers/testing.py +206 -0
  206. data/vendor/pygments-main/pygments/lexers/text.py +25 -0
  207. data/vendor/pygments-main/pygments/lexers/textedit.py +168 -0
  208. data/vendor/pygments-main/pygments/lexers/textfmts.py +429 -0
  209. data/vendor/pygments-main/pygments/lexers/theorem.py +476 -0
  210. data/vendor/pygments-main/pygments/lexers/tnt.py +262 -0
  211. data/vendor/pygments-main/pygments/lexers/trafficscript.py +53 -0
  212. data/vendor/pygments-main/pygments/lexers/typoscript.py +218 -0
  213. data/vendor/pygments-main/pygments/lexers/unicon.py +411 -0
  214. data/vendor/pygments-main/pygments/lexers/urbi.py +145 -0
  215. data/vendor/pygments-main/pygments/lexers/usd.py +89 -0
  216. data/vendor/pygments-main/pygments/lexers/varnish.py +189 -0
  217. data/vendor/pygments-main/pygments/lexers/verification.py +113 -0
  218. data/vendor/pygments-main/pygments/lexers/web.py +23 -0
  219. data/vendor/pygments-main/pygments/lexers/webidl.py +298 -0
  220. data/vendor/pygments-main/pygments/lexers/webmisc.py +991 -0
  221. data/vendor/pygments-main/pygments/lexers/whiley.py +115 -0
  222. data/vendor/pygments-main/pygments/lexers/x10.py +68 -0
  223. data/vendor/pygments-main/pygments/lexers/xorg.py +36 -0
  224. data/vendor/pygments-main/pygments/lexers/yang.py +103 -0
  225. data/vendor/pygments-main/pygments/lexers/zig.py +123 -0
  226. data/vendor/pygments-main/pygments/modeline.py +1 -2
  227. data/vendor/pygments-main/pygments/plugin.py +4 -3
  228. data/vendor/pygments-main/pygments/regexopt.py +1 -2
  229. data/vendor/pygments-main/pygments/scanner.py +2 -3
  230. data/vendor/pygments-main/pygments/sphinxext.py +2 -5
  231. data/vendor/pygments-main/pygments/style.py +61 -25
  232. data/vendor/pygments-main/pygments/styles/__init__.py +12 -5
  233. data/vendor/pygments-main/pygments/styles/abap.py +1 -2
  234. data/vendor/pygments-main/pygments/styles/algol.py +1 -2
  235. data/vendor/pygments-main/pygments/styles/algol_nu.py +1 -2
  236. data/vendor/pygments-main/pygments/styles/arduino.py +2 -3
  237. data/vendor/pygments-main/pygments/styles/autumn.py +1 -2
  238. data/vendor/pygments-main/pygments/styles/borland.py +1 -2
  239. data/vendor/pygments-main/pygments/styles/bw.py +1 -2
  240. data/vendor/pygments-main/pygments/styles/colorful.py +1 -2
  241. data/vendor/pygments-main/pygments/styles/default.py +1 -2
  242. data/vendor/pygments-main/pygments/styles/emacs.py +1 -2
  243. data/vendor/pygments-main/pygments/styles/friendly.py +1 -2
  244. data/vendor/pygments-main/pygments/styles/fruity.py +1 -2
  245. data/vendor/pygments-main/pygments/styles/igor.py +1 -2
  246. data/vendor/pygments-main/pygments/styles/inkpot.py +66 -0
  247. data/vendor/pygments-main/pygments/styles/lovelace.py +1 -2
  248. data/vendor/pygments-main/pygments/styles/manni.py +1 -2
  249. data/vendor/pygments-main/pygments/styles/material.py +118 -0
  250. data/vendor/pygments-main/pygments/styles/monokai.py +4 -4
  251. data/vendor/pygments-main/pygments/styles/murphy.py +1 -2
  252. data/vendor/pygments-main/pygments/styles/native.py +1 -2
  253. data/vendor/pygments-main/pygments/styles/paraiso_dark.py +1 -2
  254. data/vendor/pygments-main/pygments/styles/paraiso_light.py +1 -2
  255. data/vendor/pygments-main/pygments/styles/pastie.py +1 -2
  256. data/vendor/pygments-main/pygments/styles/perldoc.py +1 -2
  257. data/vendor/pygments-main/pygments/styles/rainbow_dash.py +1 -2
  258. data/vendor/pygments-main/pygments/styles/rrt.py +1 -2
  259. data/vendor/pygments-main/pygments/styles/sas.py +1 -2
  260. data/vendor/pygments-main/pygments/styles/solarized.py +136 -0
  261. data/vendor/pygments-main/pygments/styles/stata_dark.py +40 -0
  262. data/vendor/pygments-main/pygments/styles/{stata.py → stata_light.py} +14 -16
  263. data/vendor/pygments-main/pygments/styles/tango.py +1 -2
  264. data/vendor/pygments-main/pygments/styles/trac.py +1 -2
  265. data/vendor/pygments-main/pygments/styles/vim.py +1 -2
  266. data/vendor/pygments-main/pygments/styles/vs.py +1 -2
  267. data/vendor/pygments-main/pygments/styles/xcode.py +1 -2
  268. data/vendor/pygments-main/pygments/styles/zenburn.py +80 -0
  269. data/vendor/pygments-main/pygments/token.py +1 -2
  270. data/vendor/pygments-main/pygments/unistring.py +47 -109
  271. data/vendor/pygments-main/pygments/util.py +16 -96
  272. metadata +81 -139
  273. data/CHANGELOG.md +0 -111
  274. data/README.md +0 -121
  275. data/cache-lexers.rb +0 -8
  276. data/circle.yml +0 -20
  277. data/test/test_data.c +0 -2581
  278. data/test/test_data.py +0 -514
  279. data/test/test_data_generated +0 -2582
  280. data/vendor/custom_lexers/github.py +0 -565
  281. data/vendor/pygments-main/CHANGES +0 -1186
  282. data/vendor/pygments-main/MANIFEST.in +0 -6
  283. data/vendor/pygments-main/Makefile +0 -65
  284. data/vendor/pygments-main/README.rst +0 -39
  285. data/vendor/pygments-main/REVISION +0 -1
  286. data/vendor/pygments-main/TODO +0 -12
  287. data/vendor/pygments-main/doc/Makefile +0 -153
  288. data/vendor/pygments-main/doc/_static/favicon.ico +0 -0
  289. data/vendor/pygments-main/doc/_static/logo_new.png +0 -0
  290. data/vendor/pygments-main/doc/_static/logo_only.png +0 -0
  291. data/vendor/pygments-main/doc/_templates/docssidebar.html +0 -3
  292. data/vendor/pygments-main/doc/_templates/indexsidebar.html +0 -25
  293. data/vendor/pygments-main/doc/_themes/pygments14/layout.html +0 -98
  294. data/vendor/pygments-main/doc/_themes/pygments14/static/bodybg.png +0 -0
  295. data/vendor/pygments-main/doc/_themes/pygments14/static/docbg.png +0 -0
  296. data/vendor/pygments-main/doc/_themes/pygments14/static/listitem.png +0 -0
  297. data/vendor/pygments-main/doc/_themes/pygments14/static/logo.png +0 -0
  298. data/vendor/pygments-main/doc/_themes/pygments14/static/pocoo.png +0 -0
  299. data/vendor/pygments-main/doc/_themes/pygments14/static/pygments14.css_t +0 -401
  300. data/vendor/pygments-main/doc/_themes/pygments14/theme.conf +0 -15
  301. data/vendor/pygments-main/doc/conf.py +0 -241
  302. data/vendor/pygments-main/doc/docs/api.rst +0 -354
  303. data/vendor/pygments-main/doc/docs/authors.rst +0 -4
  304. data/vendor/pygments-main/doc/docs/changelog.rst +0 -1
  305. data/vendor/pygments-main/doc/docs/cmdline.rst +0 -166
  306. data/vendor/pygments-main/doc/docs/filterdevelopment.rst +0 -71
  307. data/vendor/pygments-main/doc/docs/filters.rst +0 -41
  308. data/vendor/pygments-main/doc/docs/formatterdevelopment.rst +0 -169
  309. data/vendor/pygments-main/doc/docs/formatters.rst +0 -48
  310. data/vendor/pygments-main/doc/docs/index.rst +0 -66
  311. data/vendor/pygments-main/doc/docs/integrate.rst +0 -40
  312. data/vendor/pygments-main/doc/docs/java.rst +0 -70
  313. data/vendor/pygments-main/doc/docs/lexerdevelopment.rst +0 -728
  314. data/vendor/pygments-main/doc/docs/lexers.rst +0 -69
  315. data/vendor/pygments-main/doc/docs/moinmoin.rst +0 -39
  316. data/vendor/pygments-main/doc/docs/plugins.rst +0 -93
  317. data/vendor/pygments-main/doc/docs/quickstart.rst +0 -205
  318. data/vendor/pygments-main/doc/docs/rstdirective.rst +0 -22
  319. data/vendor/pygments-main/doc/docs/styles.rst +0 -201
  320. data/vendor/pygments-main/doc/docs/tokens.rst +0 -372
  321. data/vendor/pygments-main/doc/docs/unicode.rst +0 -58
  322. data/vendor/pygments-main/doc/download.rst +0 -41
  323. data/vendor/pygments-main/doc/faq.rst +0 -139
  324. data/vendor/pygments-main/doc/index.rst +0 -54
  325. data/vendor/pygments-main/doc/languages.rst +0 -154
  326. data/vendor/pygments-main/doc/make.bat +0 -190
  327. data/vendor/pygments-main/doc/pygmentize.1 +0 -94
  328. data/vendor/pygments-main/external/autopygmentize +0 -101
  329. data/vendor/pygments-main/external/lasso-builtins-generator-9.lasso +0 -162
  330. data/vendor/pygments-main/external/markdown-processor.py +0 -67
  331. data/vendor/pygments-main/external/moin-parser.py +0 -112
  332. data/vendor/pygments-main/external/pygments.bashcomp +0 -38
  333. data/vendor/pygments-main/external/rst-directive.py +0 -82
  334. data/vendor/pygments-main/pygmentize +0 -8
  335. data/vendor/pygments-main/requirements.txt +0 -5
  336. data/vendor/pygments-main/scripts/check_sources.py +0 -211
  337. data/vendor/pygments-main/scripts/debug_lexer.py +0 -246
  338. data/vendor/pygments-main/scripts/detect_missing_analyse_text.py +0 -33
  339. data/vendor/pygments-main/scripts/epydoc.css +0 -280
  340. data/vendor/pygments-main/scripts/get_vimkw.py +0 -74
  341. data/vendor/pygments-main/scripts/pylintrc +0 -301
  342. data/vendor/pygments-main/scripts/vim2pygments.py +0 -935
  343. data/vendor/pygments-main/setup.cfg +0 -10
  344. data/vendor/pygments-main/setup.py +0 -77
  345. data/vendor/pygments-main/tox.ini +0 -7
  346. data/vendor/simplejson/.gitignore +0 -10
  347. data/vendor/simplejson/.travis.yml +0 -5
  348. data/vendor/simplejson/CHANGES.txt +0 -291
  349. data/vendor/simplejson/LICENSE.txt +0 -19
  350. data/vendor/simplejson/MANIFEST.in +0 -5
  351. data/vendor/simplejson/README.rst +0 -19
  352. data/vendor/simplejson/conf.py +0 -179
  353. data/vendor/simplejson/index.rst +0 -628
  354. data/vendor/simplejson/scripts/make_docs.py +0 -18
  355. data/vendor/simplejson/setup.py +0 -104
  356. data/vendor/simplejson/simplejson/__init__.py +0 -510
  357. data/vendor/simplejson/simplejson/_speedups.c +0 -2745
  358. data/vendor/simplejson/simplejson/decoder.py +0 -425
  359. data/vendor/simplejson/simplejson/encoder.py +0 -567
  360. data/vendor/simplejson/simplejson/ordered_dict.py +0 -119
  361. data/vendor/simplejson/simplejson/scanner.py +0 -77
  362. data/vendor/simplejson/simplejson/tests/__init__.py +0 -67
  363. data/vendor/simplejson/simplejson/tests/test_bigint_as_string.py +0 -55
  364. data/vendor/simplejson/simplejson/tests/test_check_circular.py +0 -30
  365. data/vendor/simplejson/simplejson/tests/test_decimal.py +0 -66
  366. data/vendor/simplejson/simplejson/tests/test_decode.py +0 -83
  367. data/vendor/simplejson/simplejson/tests/test_default.py +0 -9
  368. data/vendor/simplejson/simplejson/tests/test_dump.py +0 -67
  369. data/vendor/simplejson/simplejson/tests/test_encode_basestring_ascii.py +0 -46
  370. data/vendor/simplejson/simplejson/tests/test_encode_for_html.py +0 -32
  371. data/vendor/simplejson/simplejson/tests/test_errors.py +0 -34
  372. data/vendor/simplejson/simplejson/tests/test_fail.py +0 -91
  373. data/vendor/simplejson/simplejson/tests/test_float.py +0 -19
  374. data/vendor/simplejson/simplejson/tests/test_indent.py +0 -86
  375. data/vendor/simplejson/simplejson/tests/test_item_sort_key.py +0 -20
  376. data/vendor/simplejson/simplejson/tests/test_namedtuple.py +0 -121
  377. data/vendor/simplejson/simplejson/tests/test_pass1.py +0 -76
  378. data/vendor/simplejson/simplejson/tests/test_pass2.py +0 -14
  379. data/vendor/simplejson/simplejson/tests/test_pass3.py +0 -20
  380. data/vendor/simplejson/simplejson/tests/test_recursion.py +0 -67
  381. data/vendor/simplejson/simplejson/tests/test_scanstring.py +0 -117
  382. data/vendor/simplejson/simplejson/tests/test_separators.py +0 -42
  383. data/vendor/simplejson/simplejson/tests/test_speedups.py +0 -20
  384. data/vendor/simplejson/simplejson/tests/test_tuple.py +0 -49
  385. data/vendor/simplejson/simplejson/tests/test_unicode.py +0 -109
  386. data/vendor/simplejson/simplejson/tool.py +0 -39
@@ -0,0 +1,138 @@
1
+ """
2
+ pygments.lexers.ride
3
+ ~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for the Ride programming language.
6
+
7
+ :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import RegexLexer, words, include
12
+ from pygments.token import Comment, Keyword, Name, Number, Punctuation, String, Text
13
+
14
+ __all__ = ['RideLexer']
15
+
16
+
17
+ class RideLexer(RegexLexer):
18
+ """
19
+ For `Ride <https://docs.wavesplatform.com/en/ride/about-ride.html>`_
20
+ source code.
21
+
22
+ .. versionadded:: 2.6
23
+ """
24
+
25
+ name = 'Ride'
26
+ aliases = ['ride']
27
+ filenames = ['*.ride']
28
+ mimetypes = ['text/x-ride']
29
+
30
+ validName = r'[a-zA-Z_][a-zA-Z0-9_\']*'
31
+
32
+ builtinOps = (
33
+ '||', '|', '>=', '>', '==', '!',
34
+ '=', '<=', '<', '::', ':+', ':', '!=', '/',
35
+ '.', '=>', '-', '+', '*', '&&', '%', '++',
36
+ )
37
+
38
+ globalVariablesName = (
39
+ 'NOALG', 'MD5', 'SHA1', 'SHA224', 'SHA256', 'SHA384', 'SHA512',
40
+ 'SHA3224', 'SHA3256', 'SHA3384', 'SHA3512', 'nil', 'this', 'unit',
41
+ 'height', 'lastBlock', 'Buy', 'Sell', 'CEILING', 'FLOOR', 'DOWN',
42
+ 'HALFDOWN', 'HALFEVEN', 'HALFUP', 'UP',
43
+ )
44
+
45
+ typesName = (
46
+ 'Unit', 'Int', 'Boolean', 'ByteVector', 'String', 'Address', 'Alias',
47
+ 'Transfer', 'AssetPair', 'DataEntry', 'Order', 'Transaction',
48
+ 'GenesisTransaction', 'PaymentTransaction', 'ReissueTransaction',
49
+ 'BurnTransaction', 'MassTransferTransaction', 'ExchangeTransaction',
50
+ 'TransferTransaction', 'SetAssetScriptTransaction',
51
+ 'InvokeScriptTransaction', 'IssueTransaction', 'LeaseTransaction',
52
+ 'LeaseCancelTransaction', 'CreateAliasTransaction',
53
+ 'SetScriptTransaction', 'SponsorFeeTransaction', 'DataTransaction',
54
+ 'WriteSet', 'AttachedPayment', 'ScriptTransfer', 'TransferSet',
55
+ 'ScriptResult', 'Invocation', 'Asset', 'BlockInfo', 'Issue', 'Reissue',
56
+ 'Burn', 'NoAlg', 'Md5', 'Sha1', 'Sha224', 'Sha256', 'Sha384', 'Sha512',
57
+ 'Sha3224', 'Sha3256', 'Sha3384', 'Sha3512', 'BinaryEntry',
58
+ 'BooleanEntry', 'IntegerEntry', 'StringEntry', 'List', 'Ceiling',
59
+ 'Down', 'Floor', 'HalfDown', 'HalfEven', 'HalfUp', 'Up',
60
+ )
61
+
62
+ functionsName = (
63
+ 'fraction', 'size', 'toBytes', 'take', 'drop', 'takeRight', 'dropRight',
64
+ 'toString', 'isDefined', 'extract', 'throw', 'getElement', 'value',
65
+ 'cons', 'toUtf8String', 'toInt', 'indexOf', 'lastIndexOf', 'split',
66
+ 'parseInt', 'parseIntValue', 'keccak256', 'blake2b256', 'sha256',
67
+ 'sigVerify', 'toBase58String', 'fromBase58String', 'toBase64String',
68
+ 'fromBase64String', 'transactionById', 'transactionHeightById',
69
+ 'getInteger', 'getBoolean', 'getBinary', 'getString',
70
+ 'addressFromPublicKey', 'addressFromString', 'addressFromRecipient',
71
+ 'assetBalance', 'wavesBalance', 'getIntegerValue', 'getBooleanValue',
72
+ 'getBinaryValue', 'getStringValue', 'addressFromStringValue',
73
+ 'assetInfo', 'rsaVerify', 'checkMerkleProof', 'median',
74
+ 'valueOrElse', 'valueOrErrorMessage', 'contains', 'log', 'pow',
75
+ 'toBase16String', 'fromBase16String', 'blockInfoByHeight',
76
+ 'transferTransactionById',
77
+ )
78
+
79
+ reservedWords = words((
80
+ 'match', 'case', 'else', 'func', 'if',
81
+ 'let', 'then', '@Callable', '@Verifier',
82
+ ), suffix=r'\b')
83
+
84
+ tokens = {
85
+ 'root': [
86
+ # Comments
87
+ (r'#.*', Comment.Single),
88
+ # Whitespace
89
+ (r'\s+', Text),
90
+ # Strings
91
+ (r'"', String, 'doublequote'),
92
+ (r'utf8\'', String, 'utf8quote'),
93
+ (r'base(58|64|16)\'', String, 'singlequote'),
94
+ # Keywords
95
+ (reservedWords, Keyword.Reserved),
96
+ (r'\{-#.*?#-\}', Keyword.Reserved),
97
+ (r'FOLD<\d+>', Keyword.Reserved),
98
+ # Types
99
+ (words(typesName), Keyword.Type),
100
+ # Main
101
+ # (specialName, Keyword.Reserved),
102
+ # Prefix Operators
103
+ (words(builtinOps, prefix=r'\(', suffix=r'\)'), Name.Function),
104
+ # Infix Operators
105
+ (words(builtinOps), Name.Function),
106
+ (words(globalVariablesName), Name.Function),
107
+ (words(functionsName), Name.Function),
108
+ # Numbers
109
+ include('numbers'),
110
+ # Variable Names
111
+ (validName, Name.Variable),
112
+ # Parens
113
+ (r'[,()\[\]{}]', Punctuation),
114
+ ],
115
+
116
+ 'doublequote': [
117
+ (r'\\u[0-9a-fA-F]{4}', String.Escape),
118
+ (r'\\[nrfvb\\"]', String.Escape),
119
+ (r'[^"]', String),
120
+ (r'"', String, '#pop'),
121
+ ],
122
+
123
+ 'utf8quote': [
124
+ (r'\\u[0-9a-fA-F]{4}', String.Escape),
125
+ (r'\\[nrfvb\\\']', String.Escape),
126
+ (r'[^\']', String),
127
+ (r'\'', String, '#pop'),
128
+ ],
129
+
130
+ 'singlequote': [
131
+ (r'[^\']', String),
132
+ (r'\'', String, '#pop'),
133
+ ],
134
+
135
+ 'numbers': [
136
+ (r'_?\d+', Number.Integer),
137
+ ],
138
+ }
@@ -0,0 +1,66 @@
1
+ """
2
+ pygments.lexers.rnc
3
+ ~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for Relax-NG Compact syntax
6
+
7
+ :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import RegexLexer
12
+ from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
13
+ Punctuation
14
+
15
+ __all__ = ['RNCCompactLexer']
16
+
17
+
18
+ class RNCCompactLexer(RegexLexer):
19
+ """
20
+ For `RelaxNG-compact <http://relaxng.org>`_ syntax.
21
+
22
+ .. versionadded:: 2.2
23
+ """
24
+
25
+ name = 'Relax-NG Compact'
26
+ aliases = ['rnc', 'rng-compact']
27
+ filenames = ['*.rnc']
28
+
29
+ tokens = {
30
+ 'root': [
31
+ (r'namespace\b', Keyword.Namespace),
32
+ (r'(?:default|datatypes)\b', Keyword.Declaration),
33
+ (r'##.*$', Comment.Preproc),
34
+ (r'#.*$', Comment.Single),
35
+ (r'"[^"]*"', String.Double),
36
+ # TODO single quoted strings and escape sequences outside of
37
+ # double-quoted strings
38
+ (r'(?:element|attribute|mixed)\b', Keyword.Declaration, 'variable'),
39
+ (r'(text\b|xsd:[^ ]+)', Keyword.Type, 'maybe_xsdattributes'),
40
+ (r'[,?&*=|~]|>>', Operator),
41
+ (r'[(){}]', Punctuation),
42
+ (r'.', Text),
43
+ ],
44
+
45
+ # a variable has been declared using `element` or `attribute`
46
+ 'variable': [
47
+ (r'[^{]+', Name.Variable),
48
+ (r'\{', Punctuation, '#pop'),
49
+ ],
50
+
51
+ # after an xsd:<datatype> declaration there may be attributes
52
+ 'maybe_xsdattributes': [
53
+ (r'\{', Punctuation, 'xsdattributes'),
54
+ (r'\}', Punctuation, '#pop'),
55
+ (r'.', Text),
56
+ ],
57
+
58
+ # attributes take the form { key1 = value1 key2 = value2 ... }
59
+ 'xsdattributes': [
60
+ (r'[^ =}]', Name.Attribute),
61
+ (r'=', Operator),
62
+ (r'"[^"]*"', String.Double),
63
+ (r'\}', Punctuation, '#pop'),
64
+ (r'.', Text),
65
+ ],
66
+ }
@@ -0,0 +1,81 @@
1
+ """
2
+ pygments.lexers.roboconf
3
+ ~~~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexers for Roboconf DSL.
6
+
7
+ :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import RegexLexer, words, re
12
+ from pygments.token import Text, Operator, Keyword, Name, Comment
13
+
14
+ __all__ = ['RoboconfGraphLexer', 'RoboconfInstancesLexer']
15
+
16
+
17
+ class RoboconfGraphLexer(RegexLexer):
18
+ """
19
+ Lexer for `Roboconf <http://roboconf.net/en/roboconf.html>`_ graph files.
20
+
21
+ .. versionadded:: 2.1
22
+ """
23
+ name = 'Roboconf Graph'
24
+ aliases = ['roboconf-graph']
25
+ filenames = ['*.graph']
26
+
27
+ flags = re.IGNORECASE | re.MULTILINE
28
+ tokens = {
29
+ 'root': [
30
+ # Skip white spaces
31
+ (r'\s+', Text),
32
+
33
+ # There is one operator
34
+ (r'=', Operator),
35
+
36
+ # Keywords
37
+ (words(('facet', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword),
38
+ (words((
39
+ 'installer', 'extends', 'exports', 'imports', 'facets',
40
+ 'children'), suffix=r'\s*:?', prefix=r'\b'), Name),
41
+
42
+ # Comments
43
+ (r'#.*\n', Comment),
44
+
45
+ # Default
46
+ (r'[^#]', Text),
47
+ (r'.*\n', Text)
48
+ ]
49
+ }
50
+
51
+
52
+ class RoboconfInstancesLexer(RegexLexer):
53
+ """
54
+ Lexer for `Roboconf <http://roboconf.net/en/roboconf.html>`_ instances files.
55
+
56
+ .. versionadded:: 2.1
57
+ """
58
+ name = 'Roboconf Instances'
59
+ aliases = ['roboconf-instances']
60
+ filenames = ['*.instances']
61
+
62
+ flags = re.IGNORECASE | re.MULTILINE
63
+ tokens = {
64
+ 'root': [
65
+
66
+ # Skip white spaces
67
+ (r'\s+', Text),
68
+
69
+ # Keywords
70
+ (words(('instance of', 'import'), suffix=r'\s*\b', prefix=r'\b'), Keyword),
71
+ (words(('name', 'count'), suffix=r's*:?', prefix=r'\b'), Name),
72
+ (r'\s*[\w.-]+\s*:', Name),
73
+
74
+ # Comments
75
+ (r'#.*\n', Comment),
76
+
77
+ # Default
78
+ (r'[^#]', Text),
79
+ (r'.*\n', Text)
80
+ ]
81
+ }
@@ -0,0 +1,551 @@
1
+ """
2
+ pygments.lexers.robotframework
3
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for Robot Framework.
6
+
7
+ :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ # Copyright 2012 Nokia Siemens Networks Oyj
12
+ #
13
+ # Licensed under the Apache License, Version 2.0 (the "License");
14
+ # you may not use this file except in compliance with the License.
15
+ # You may obtain a copy of the License at
16
+ #
17
+ # http://www.apache.org/licenses/LICENSE-2.0
18
+ #
19
+ # Unless required by applicable law or agreed to in writing, software
20
+ # distributed under the License is distributed on an "AS IS" BASIS,
21
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22
+ # See the License for the specific language governing permissions and
23
+ # limitations under the License.
24
+
25
+ import re
26
+
27
+ from pygments.lexer import Lexer
28
+ from pygments.token import Token
29
+
30
+ __all__ = ['RobotFrameworkLexer']
31
+
32
+
33
+ HEADING = Token.Generic.Heading
34
+ SETTING = Token.Keyword.Namespace
35
+ IMPORT = Token.Name.Namespace
36
+ TC_KW_NAME = Token.Generic.Subheading
37
+ KEYWORD = Token.Name.Function
38
+ ARGUMENT = Token.String
39
+ VARIABLE = Token.Name.Variable
40
+ COMMENT = Token.Comment
41
+ SEPARATOR = Token.Punctuation
42
+ SYNTAX = Token.Punctuation
43
+ GHERKIN = Token.Generic.Emph
44
+ ERROR = Token.Error
45
+
46
+
47
+ def normalize(string, remove=''):
48
+ string = string.lower()
49
+ for char in remove + ' ':
50
+ if char in string:
51
+ string = string.replace(char, '')
52
+ return string
53
+
54
+
55
+ class RobotFrameworkLexer(Lexer):
56
+ """
57
+ For `Robot Framework <http://robotframework.org>`_ test data.
58
+
59
+ Supports both space and pipe separated plain text formats.
60
+
61
+ .. versionadded:: 1.6
62
+ """
63
+ name = 'RobotFramework'
64
+ aliases = ['robotframework']
65
+ filenames = ['*.robot']
66
+ mimetypes = ['text/x-robotframework']
67
+
68
+ def __init__(self, **options):
69
+ options['tabsize'] = 2
70
+ options['encoding'] = 'UTF-8'
71
+ Lexer.__init__(self, **options)
72
+
73
+ def get_tokens_unprocessed(self, text):
74
+ row_tokenizer = RowTokenizer()
75
+ var_tokenizer = VariableTokenizer()
76
+ index = 0
77
+ for row in text.splitlines():
78
+ for value, token in row_tokenizer.tokenize(row):
79
+ for value, token in var_tokenizer.tokenize(value, token):
80
+ if value:
81
+ yield index, token, str(value)
82
+ index += len(value)
83
+
84
+
85
+ class VariableTokenizer:
86
+
87
+ def tokenize(self, string, token):
88
+ var = VariableSplitter(string, identifiers='$@%&')
89
+ if var.start < 0 or token in (COMMENT, ERROR):
90
+ yield string, token
91
+ return
92
+ for value, token in self._tokenize(var, string, token):
93
+ if value:
94
+ yield value, token
95
+
96
+ def _tokenize(self, var, string, orig_token):
97
+ before = string[:var.start]
98
+ yield before, orig_token
99
+ yield var.identifier + '{', SYNTAX
100
+ yield from self.tokenize(var.base, VARIABLE)
101
+ yield '}', SYNTAX
102
+ if var.index:
103
+ yield '[', SYNTAX
104
+ yield from self.tokenize(var.index, VARIABLE)
105
+ yield ']', SYNTAX
106
+ yield from self.tokenize(string[var.end:], orig_token)
107
+
108
+
109
+ class RowTokenizer:
110
+
111
+ def __init__(self):
112
+ self._table = UnknownTable()
113
+ self._splitter = RowSplitter()
114
+ testcases = TestCaseTable()
115
+ settings = SettingTable(testcases.set_default_template)
116
+ variables = VariableTable()
117
+ keywords = KeywordTable()
118
+ self._tables = {'settings': settings, 'setting': settings,
119
+ 'metadata': settings,
120
+ 'variables': variables, 'variable': variables,
121
+ 'testcases': testcases, 'testcase': testcases,
122
+ 'tasks': testcases, 'task': testcases,
123
+ 'keywords': keywords, 'keyword': keywords,
124
+ 'userkeywords': keywords, 'userkeyword': keywords}
125
+
126
+ def tokenize(self, row):
127
+ commented = False
128
+ heading = False
129
+ for index, value in enumerate(self._splitter.split(row)):
130
+ # First value, and every second after that, is a separator.
131
+ index, separator = divmod(index-1, 2)
132
+ if value.startswith('#'):
133
+ commented = True
134
+ elif index == 0 and value.startswith('*'):
135
+ self._table = self._start_table(value)
136
+ heading = True
137
+ yield from self._tokenize(value, index, commented,
138
+ separator, heading)
139
+ self._table.end_row()
140
+
141
+ def _start_table(self, header):
142
+ name = normalize(header, remove='*')
143
+ return self._tables.get(name, UnknownTable())
144
+
145
+ def _tokenize(self, value, index, commented, separator, heading):
146
+ if commented:
147
+ yield value, COMMENT
148
+ elif separator:
149
+ yield value, SEPARATOR
150
+ elif heading:
151
+ yield value, HEADING
152
+ else:
153
+ yield from self._table.tokenize(value, index)
154
+
155
+
156
+ class RowSplitter:
157
+ _space_splitter = re.compile('( {2,})')
158
+ _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))')
159
+
160
+ def split(self, row):
161
+ splitter = (row.startswith('| ') and self._split_from_pipes
162
+ or self._split_from_spaces)
163
+ yield from splitter(row)
164
+ yield '\n'
165
+
166
+ def _split_from_spaces(self, row):
167
+ yield '' # Start with (pseudo)separator similarly as with pipes
168
+ yield from self._space_splitter.split(row)
169
+
170
+ def _split_from_pipes(self, row):
171
+ _, separator, rest = self._pipe_splitter.split(row, 1)
172
+ yield separator
173
+ while self._pipe_splitter.search(rest):
174
+ cell, separator, rest = self._pipe_splitter.split(rest, 1)
175
+ yield cell
176
+ yield separator
177
+ yield rest
178
+
179
+
180
+ class Tokenizer:
181
+ _tokens = None
182
+
183
+ def __init__(self):
184
+ self._index = 0
185
+
186
+ def tokenize(self, value):
187
+ values_and_tokens = self._tokenize(value, self._index)
188
+ self._index += 1
189
+ if isinstance(values_and_tokens, type(Token)):
190
+ values_and_tokens = [(value, values_and_tokens)]
191
+ return values_and_tokens
192
+
193
+ def _tokenize(self, value, index):
194
+ index = min(index, len(self._tokens) - 1)
195
+ return self._tokens[index]
196
+
197
+ def _is_assign(self, value):
198
+ if value.endswith('='):
199
+ value = value[:-1].strip()
200
+ var = VariableSplitter(value, identifiers='$@&')
201
+ return var.start == 0 and var.end == len(value)
202
+
203
+
204
+ class Comment(Tokenizer):
205
+ _tokens = (COMMENT,)
206
+
207
+
208
+ class Setting(Tokenizer):
209
+ _tokens = (SETTING, ARGUMENT)
210
+ _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown',
211
+ 'suitepostcondition', 'testsetup', 'tasksetup', 'testprecondition',
212
+ 'testteardown','taskteardown', 'testpostcondition', 'testtemplate', 'tasktemplate')
213
+ _import_settings = ('library', 'resource', 'variables')
214
+ _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags',
215
+ 'testtimeout','tasktimeout')
216
+ _custom_tokenizer = None
217
+
218
+ def __init__(self, template_setter=None):
219
+ Tokenizer.__init__(self)
220
+ self._template_setter = template_setter
221
+
222
+ def _tokenize(self, value, index):
223
+ if index == 1 and self._template_setter:
224
+ self._template_setter(value)
225
+ if index == 0:
226
+ normalized = normalize(value)
227
+ if normalized in self._keyword_settings:
228
+ self._custom_tokenizer = KeywordCall(support_assign=False)
229
+ elif normalized in self._import_settings:
230
+ self._custom_tokenizer = ImportSetting()
231
+ elif normalized not in self._other_settings:
232
+ return ERROR
233
+ elif self._custom_tokenizer:
234
+ return self._custom_tokenizer.tokenize(value)
235
+ return Tokenizer._tokenize(self, value, index)
236
+
237
+
238
+ class ImportSetting(Tokenizer):
239
+ _tokens = (IMPORT, ARGUMENT)
240
+
241
+
242
+ class TestCaseSetting(Setting):
243
+ _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition',
244
+ 'template')
245
+ _import_settings = ()
246
+ _other_settings = ('documentation', 'tags', 'timeout')
247
+
248
+ def _tokenize(self, value, index):
249
+ if index == 0:
250
+ type = Setting._tokenize(self, value[1:-1], index)
251
+ return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)]
252
+ return Setting._tokenize(self, value, index)
253
+
254
+
255
+ class KeywordSetting(TestCaseSetting):
256
+ _keyword_settings = ('teardown',)
257
+ _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags')
258
+
259
+
260
+ class Variable(Tokenizer):
261
+ _tokens = (SYNTAX, ARGUMENT)
262
+
263
+ def _tokenize(self, value, index):
264
+ if index == 0 and not self._is_assign(value):
265
+ return ERROR
266
+ return Tokenizer._tokenize(self, value, index)
267
+
268
+
269
+ class KeywordCall(Tokenizer):
270
+ _tokens = (KEYWORD, ARGUMENT)
271
+
272
+ def __init__(self, support_assign=True):
273
+ Tokenizer.__init__(self)
274
+ self._keyword_found = not support_assign
275
+ self._assigns = 0
276
+
277
+ def _tokenize(self, value, index):
278
+ if not self._keyword_found and self._is_assign(value):
279
+ self._assigns += 1
280
+ return SYNTAX # VariableTokenizer tokenizes this later.
281
+ if self._keyword_found:
282
+ return Tokenizer._tokenize(self, value, index - self._assigns)
283
+ self._keyword_found = True
284
+ return GherkinTokenizer().tokenize(value, KEYWORD)
285
+
286
+
287
+ class GherkinTokenizer:
288
+ _gherkin_prefix = re.compile('^(Given|When|Then|And) ', re.IGNORECASE)
289
+
290
+ def tokenize(self, value, token):
291
+ match = self._gherkin_prefix.match(value)
292
+ if not match:
293
+ return [(value, token)]
294
+ end = match.end()
295
+ return [(value[:end], GHERKIN), (value[end:], token)]
296
+
297
+
298
+ class TemplatedKeywordCall(Tokenizer):
299
+ _tokens = (ARGUMENT,)
300
+
301
+
302
+ class ForLoop(Tokenizer):
303
+
304
+ def __init__(self):
305
+ Tokenizer.__init__(self)
306
+ self._in_arguments = False
307
+
308
+ def _tokenize(self, value, index):
309
+ token = self._in_arguments and ARGUMENT or SYNTAX
310
+ if value.upper() in ('IN', 'IN RANGE'):
311
+ self._in_arguments = True
312
+ return token
313
+
314
+
315
+ class _Table:
316
+ _tokenizer_class = None
317
+
318
+ def __init__(self, prev_tokenizer=None):
319
+ self._tokenizer = self._tokenizer_class()
320
+ self._prev_tokenizer = prev_tokenizer
321
+ self._prev_values_on_row = []
322
+
323
+ def tokenize(self, value, index):
324
+ if self._continues(value, index):
325
+ self._tokenizer = self._prev_tokenizer
326
+ yield value, SYNTAX
327
+ else:
328
+ yield from self._tokenize(value, index)
329
+ self._prev_values_on_row.append(value)
330
+
331
+ def _continues(self, value, index):
332
+ return value == '...' and all(self._is_empty(t)
333
+ for t in self._prev_values_on_row)
334
+
335
+ def _is_empty(self, value):
336
+ return value in ('', '\\')
337
+
338
+ def _tokenize(self, value, index):
339
+ return self._tokenizer.tokenize(value)
340
+
341
+ def end_row(self):
342
+ self.__init__(prev_tokenizer=self._tokenizer)
343
+
344
+
345
+ class UnknownTable(_Table):
346
+ _tokenizer_class = Comment
347
+
348
+ def _continues(self, value, index):
349
+ return False
350
+
351
+
352
+ class VariableTable(_Table):
353
+ _tokenizer_class = Variable
354
+
355
+
356
+ class SettingTable(_Table):
357
+ _tokenizer_class = Setting
358
+
359
+ def __init__(self, template_setter, prev_tokenizer=None):
360
+ _Table.__init__(self, prev_tokenizer)
361
+ self._template_setter = template_setter
362
+
363
+ def _tokenize(self, value, index):
364
+ if index == 0 and normalize(value) == 'testtemplate':
365
+ self._tokenizer = Setting(self._template_setter)
366
+ return _Table._tokenize(self, value, index)
367
+
368
+ def end_row(self):
369
+ self.__init__(self._template_setter, prev_tokenizer=self._tokenizer)
370
+
371
+
372
+ class TestCaseTable(_Table):
373
+ _setting_class = TestCaseSetting
374
+ _test_template = None
375
+ _default_template = None
376
+
377
+ @property
378
+ def _tokenizer_class(self):
379
+ if self._test_template or (self._default_template and
380
+ self._test_template is not False):
381
+ return TemplatedKeywordCall
382
+ return KeywordCall
383
+
384
+ def _continues(self, value, index):
385
+ return index > 0 and _Table._continues(self, value, index)
386
+
387
+ def _tokenize(self, value, index):
388
+ if index == 0:
389
+ if value:
390
+ self._test_template = None
391
+ return GherkinTokenizer().tokenize(value, TC_KW_NAME)
392
+ if index == 1 and self._is_setting(value):
393
+ if self._is_template(value):
394
+ self._test_template = False
395
+ self._tokenizer = self._setting_class(self.set_test_template)
396
+ else:
397
+ self._tokenizer = self._setting_class()
398
+ if index == 1 and self._is_for_loop(value):
399
+ self._tokenizer = ForLoop()
400
+ if index == 1 and self._is_empty(value):
401
+ return [(value, SYNTAX)]
402
+ return _Table._tokenize(self, value, index)
403
+
404
+ def _is_setting(self, value):
405
+ return value.startswith('[') and value.endswith(']')
406
+
407
+ def _is_template(self, value):
408
+ return normalize(value) == '[template]'
409
+
410
+ def _is_for_loop(self, value):
411
+ return value.startswith(':') and normalize(value, remove=':') == 'for'
412
+
413
+ def set_test_template(self, template):
414
+ self._test_template = self._is_template_set(template)
415
+
416
+ def set_default_template(self, template):
417
+ self._default_template = self._is_template_set(template)
418
+
419
+ def _is_template_set(self, template):
420
+ return normalize(template) not in ('', '\\', 'none', '${empty}')
421
+
422
+
423
+ class KeywordTable(TestCaseTable):
424
+ _tokenizer_class = KeywordCall
425
+ _setting_class = KeywordSetting
426
+
427
+ def _is_template(self, value):
428
+ return False
429
+
430
+
431
+ # Following code copied directly from Robot Framework 2.7.5.
432
+
433
+ class VariableSplitter:
434
+
435
+ def __init__(self, string, identifiers):
436
+ self.identifier = None
437
+ self.base = None
438
+ self.index = None
439
+ self.start = -1
440
+ self.end = -1
441
+ self._identifiers = identifiers
442
+ self._may_have_internal_variables = False
443
+ try:
444
+ self._split(string)
445
+ except ValueError:
446
+ pass
447
+ else:
448
+ self._finalize()
449
+
450
+ def get_replaced_base(self, variables):
451
+ if self._may_have_internal_variables:
452
+ return variables.replace_string(self.base)
453
+ return self.base
454
+
455
+ def _finalize(self):
456
+ self.identifier = self._variable_chars[0]
457
+ self.base = ''.join(self._variable_chars[2:-1])
458
+ self.end = self.start + len(self._variable_chars)
459
+ if self._has_list_or_dict_variable_index():
460
+ self.index = ''.join(self._list_and_dict_variable_index_chars[1:-1])
461
+ self.end += len(self._list_and_dict_variable_index_chars)
462
+
463
+ def _has_list_or_dict_variable_index(self):
464
+ return self._list_and_dict_variable_index_chars\
465
+ and self._list_and_dict_variable_index_chars[-1] == ']'
466
+
467
+ def _split(self, string):
468
+ start_index, max_index = self._find_variable(string)
469
+ self.start = start_index
470
+ self._open_curly = 1
471
+ self._state = self._variable_state
472
+ self._variable_chars = [string[start_index], '{']
473
+ self._list_and_dict_variable_index_chars = []
474
+ self._string = string
475
+ start_index += 2
476
+ for index, char in enumerate(string[start_index:]):
477
+ index += start_index # Giving start to enumerate only in Py 2.6+
478
+ try:
479
+ self._state(char, index)
480
+ except StopIteration:
481
+ return
482
+ if index == max_index and not self._scanning_list_variable_index():
483
+ return
484
+
485
+ def _scanning_list_variable_index(self):
486
+ return self._state in [self._waiting_list_variable_index_state,
487
+ self._list_variable_index_state]
488
+
489
+ def _find_variable(self, string):
490
+ max_end_index = string.rfind('}')
491
+ if max_end_index == -1:
492
+ raise ValueError('No variable end found')
493
+ if self._is_escaped(string, max_end_index):
494
+ return self._find_variable(string[:max_end_index])
495
+ start_index = self._find_start_index(string, 1, max_end_index)
496
+ if start_index == -1:
497
+ raise ValueError('No variable start found')
498
+ return start_index, max_end_index
499
+
500
+ def _find_start_index(self, string, start, end):
501
+ index = string.find('{', start, end) - 1
502
+ if index < 0:
503
+ return -1
504
+ if self._start_index_is_ok(string, index):
505
+ return index
506
+ return self._find_start_index(string, index+2, end)
507
+
508
+ def _start_index_is_ok(self, string, index):
509
+ return string[index] in self._identifiers\
510
+ and not self._is_escaped(string, index)
511
+
512
+ def _is_escaped(self, string, index):
513
+ escaped = False
514
+ while index > 0 and string[index-1] == '\\':
515
+ index -= 1
516
+ escaped = not escaped
517
+ return escaped
518
+
519
+ def _variable_state(self, char, index):
520
+ self._variable_chars.append(char)
521
+ if char == '}' and not self._is_escaped(self._string, index):
522
+ self._open_curly -= 1
523
+ if self._open_curly == 0:
524
+ if not self._is_list_or_dict_variable():
525
+ raise StopIteration
526
+ self._state = self._waiting_list_variable_index_state
527
+ elif char in self._identifiers:
528
+ self._state = self._internal_variable_start_state
529
+
530
+ def _is_list_or_dict_variable(self):
531
+ return self._variable_chars[0] in ('@','&')
532
+
533
+ def _internal_variable_start_state(self, char, index):
534
+ self._state = self._variable_state
535
+ if char == '{':
536
+ self._variable_chars.append(char)
537
+ self._open_curly += 1
538
+ self._may_have_internal_variables = True
539
+ else:
540
+ self._variable_state(char, index)
541
+
542
+ def _waiting_list_variable_index_state(self, char, index):
543
+ if char != '[':
544
+ raise StopIteration
545
+ self._list_and_dict_variable_index_chars.append(char)
546
+ self._state = self._list_variable_index_state
547
+
548
+ def _list_variable_index_state(self, char, index):
549
+ self._list_and_dict_variable_index_chars.append(char)
550
+ if char == ']':
551
+ raise StopIteration