pygments.rb 2.4.1 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (348) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +8 -8
  3. data/.github/workflows/release.yml +5 -5
  4. data/.gitignore +2 -0
  5. data/.rubocop.yml +1 -1
  6. data/CHANGELOG.adoc +9 -1
  7. data/README.adoc +2 -2
  8. data/lib/pygments/version.rb +1 -1
  9. data/pygments.rb.gemspec +2 -2
  10. data/vendor/pygments-main/bin/pygmentize +1 -1
  11. data/vendor/pygments-main/pygments/__init__.py +2 -2
  12. data/vendor/pygments-main/pygments/__main__.py +1 -1
  13. data/vendor/pygments-main/pygments/cmdline.py +12 -12
  14. data/vendor/pygments-main/pygments/console.py +5 -5
  15. data/vendor/pygments-main/pygments/filter.py +2 -3
  16. data/vendor/pygments-main/pygments/filters/__init__.py +4 -4
  17. data/vendor/pygments-main/pygments/formatter.py +6 -1
  18. data/vendor/pygments-main/pygments/formatters/__init__.py +7 -8
  19. data/vendor/pygments-main/pygments/formatters/_mapping.py +0 -0
  20. data/vendor/pygments-main/pygments/formatters/bbcode.py +2 -2
  21. data/vendor/pygments-main/pygments/formatters/groff.py +3 -3
  22. data/vendor/pygments-main/pygments/formatters/html.py +38 -40
  23. data/vendor/pygments-main/pygments/formatters/img.py +51 -11
  24. data/vendor/pygments-main/pygments/formatters/irc.py +1 -1
  25. data/vendor/pygments-main/pygments/formatters/latex.py +24 -27
  26. data/vendor/pygments-main/pygments/formatters/other.py +3 -4
  27. data/vendor/pygments-main/pygments/formatters/pangomarkup.py +2 -2
  28. data/vendor/pygments-main/pygments/formatters/rtf.py +228 -25
  29. data/vendor/pygments-main/pygments/formatters/svg.py +17 -20
  30. data/vendor/pygments-main/pygments/formatters/terminal.py +1 -1
  31. data/vendor/pygments-main/pygments/formatters/terminal256.py +1 -1
  32. data/vendor/pygments-main/pygments/lexer.py +51 -33
  33. data/vendor/pygments-main/pygments/lexers/__init__.py +12 -12
  34. data/vendor/pygments-main/pygments/lexers/_ada_builtins.py +1 -1
  35. data/vendor/pygments-main/pygments/lexers/_asy_builtins.py +1 -1
  36. data/vendor/pygments-main/pygments/lexers/_cl_builtins.py +1 -1
  37. data/vendor/pygments-main/pygments/lexers/_cocoa_builtins.py +2 -2
  38. data/vendor/pygments-main/pygments/lexers/_csound_builtins.py +1 -1
  39. data/vendor/pygments-main/pygments/lexers/_css_builtins.py +1 -1
  40. data/vendor/pygments-main/pygments/lexers/_julia_builtins.py +1 -1
  41. data/vendor/pygments-main/pygments/lexers/_lasso_builtins.py +1 -1
  42. data/vendor/pygments-main/pygments/lexers/_lilypond_builtins.py +1 -1
  43. data/vendor/pygments-main/pygments/lexers/_lua_builtins.py +5 -5
  44. data/vendor/pygments-main/pygments/lexers/_luau_builtins.py +62 -0
  45. data/vendor/pygments-main/pygments/lexers/_mapping.py +41 -11
  46. data/vendor/pygments-main/pygments/lexers/_mql_builtins.py +1 -1
  47. data/vendor/pygments-main/pygments/lexers/_mysql_builtins.py +3 -3
  48. data/vendor/pygments-main/pygments/lexers/_openedge_builtins.py +1 -1
  49. data/vendor/pygments-main/pygments/lexers/_php_builtins.py +3 -3
  50. data/vendor/pygments-main/pygments/lexers/_postgres_builtins.py +5 -5
  51. data/vendor/pygments-main/pygments/lexers/_qlik_builtins.py +1 -1
  52. data/vendor/pygments-main/pygments/lexers/_scheme_builtins.py +1 -1
  53. data/vendor/pygments-main/pygments/lexers/_scilab_builtins.py +4 -4
  54. data/vendor/pygments-main/pygments/lexers/_sourcemod_builtins.py +3 -3
  55. data/vendor/pygments-main/pygments/lexers/_stan_builtins.py +1 -1
  56. data/vendor/pygments-main/pygments/lexers/_stata_builtins.py +1 -1
  57. data/vendor/pygments-main/pygments/lexers/_tsql_builtins.py +1 -1
  58. data/vendor/pygments-main/pygments/lexers/_usd_builtins.py +1 -1
  59. data/vendor/pygments-main/pygments/lexers/_vbscript_builtins.py +1 -1
  60. data/vendor/pygments-main/pygments/lexers/_vim_builtins.py +1 -1
  61. data/vendor/pygments-main/pygments/lexers/actionscript.py +6 -8
  62. data/vendor/pygments-main/pygments/lexers/ada.py +3 -3
  63. data/vendor/pygments-main/pygments/lexers/agile.py +3 -1
  64. data/vendor/pygments-main/pygments/lexers/algebra.py +9 -13
  65. data/vendor/pygments-main/pygments/lexers/ambient.py +2 -3
  66. data/vendor/pygments-main/pygments/lexers/amdgpu.py +3 -3
  67. data/vendor/pygments-main/pygments/lexers/ampl.py +2 -3
  68. data/vendor/pygments-main/pygments/lexers/apdlexer.py +4 -3
  69. data/vendor/pygments-main/pygments/lexers/apl.py +2 -3
  70. data/vendor/pygments-main/pygments/lexers/archetype.py +8 -12
  71. data/vendor/pygments-main/pygments/lexers/arrow.py +2 -3
  72. data/vendor/pygments-main/pygments/lexers/arturo.py +2 -3
  73. data/vendor/pygments-main/pygments/lexers/asc.py +4 -4
  74. data/vendor/pygments-main/pygments/lexers/asm.py +28 -15
  75. data/vendor/pygments-main/pygments/lexers/asn1.py +178 -0
  76. data/vendor/pygments-main/pygments/lexers/automation.py +6 -8
  77. data/vendor/pygments-main/pygments/lexers/bare.py +2 -3
  78. data/vendor/pygments-main/pygments/lexers/basic.py +48 -57
  79. data/vendor/pygments-main/pygments/lexers/bdd.py +3 -4
  80. data/vendor/pygments-main/pygments/lexers/berry.py +4 -4
  81. data/vendor/pygments-main/pygments/lexers/bibtex.py +5 -5
  82. data/vendor/pygments-main/pygments/lexers/blueprint.py +173 -0
  83. data/vendor/pygments-main/pygments/lexers/boa.py +4 -4
  84. data/vendor/pygments-main/pygments/lexers/bqn.py +109 -0
  85. data/vendor/pygments-main/pygments/lexers/business.py +21 -22
  86. data/vendor/pygments-main/pygments/lexers/c_cpp.py +9 -4
  87. data/vendor/pygments-main/pygments/lexers/c_like.py +99 -27
  88. data/vendor/pygments-main/pygments/lexers/capnproto.py +2 -3
  89. data/vendor/pygments-main/pygments/lexers/carbon.py +10 -11
  90. data/vendor/pygments-main/pygments/lexers/cddl.py +8 -9
  91. data/vendor/pygments-main/pygments/lexers/chapel.py +2 -3
  92. data/vendor/pygments-main/pygments/lexers/clean.py +2 -1
  93. data/vendor/pygments-main/pygments/lexers/comal.py +2 -1
  94. data/vendor/pygments-main/pygments/lexers/compiled.py +2 -1
  95. data/vendor/pygments-main/pygments/lexers/configs.py +342 -108
  96. data/vendor/pygments-main/pygments/lexers/console.py +5 -5
  97. data/vendor/pygments-main/pygments/lexers/cplint.py +2 -3
  98. data/vendor/pygments-main/pygments/lexers/crystal.py +4 -5
  99. data/vendor/pygments-main/pygments/lexers/csound.py +6 -8
  100. data/vendor/pygments-main/pygments/lexers/css.py +5 -5
  101. data/vendor/pygments-main/pygments/lexers/d.py +4 -3
  102. data/vendor/pygments-main/pygments/lexers/dalvik.py +2 -3
  103. data/vendor/pygments-main/pygments/lexers/data.py +8 -12
  104. data/vendor/pygments-main/pygments/lexers/dax.py +2 -3
  105. data/vendor/pygments-main/pygments/lexers/devicetree.py +2 -3
  106. data/vendor/pygments-main/pygments/lexers/diff.py +7 -6
  107. data/vendor/pygments-main/pygments/lexers/dns.py +109 -0
  108. data/vendor/pygments-main/pygments/lexers/dotnet.py +19 -14
  109. data/vendor/pygments-main/pygments/lexers/dsls.py +24 -35
  110. data/vendor/pygments-main/pygments/lexers/dylan.py +8 -16
  111. data/vendor/pygments-main/pygments/lexers/ecl.py +2 -3
  112. data/vendor/pygments-main/pygments/lexers/eiffel.py +3 -4
  113. data/vendor/pygments-main/pygments/lexers/elm.py +3 -4
  114. data/vendor/pygments-main/pygments/lexers/elpi.py +28 -21
  115. data/vendor/pygments-main/pygments/lexers/email.py +3 -3
  116. data/vendor/pygments-main/pygments/lexers/erlang.py +22 -24
  117. data/vendor/pygments-main/pygments/lexers/esoteric.py +11 -12
  118. data/vendor/pygments-main/pygments/lexers/ezhil.py +2 -3
  119. data/vendor/pygments-main/pygments/lexers/factor.py +2 -3
  120. data/vendor/pygments-main/pygments/lexers/fantom.py +3 -3
  121. data/vendor/pygments-main/pygments/lexers/felix.py +9 -10
  122. data/vendor/pygments-main/pygments/lexers/fift.py +2 -1
  123. data/vendor/pygments-main/pygments/lexers/floscript.py +2 -3
  124. data/vendor/pygments-main/pygments/lexers/forth.py +2 -3
  125. data/vendor/pygments-main/pygments/lexers/fortran.py +6 -7
  126. data/vendor/pygments-main/pygments/lexers/foxpro.py +3 -3
  127. data/vendor/pygments-main/pygments/lexers/freefem.py +2 -3
  128. data/vendor/pygments-main/pygments/lexers/func.py +3 -1
  129. data/vendor/pygments-main/pygments/lexers/functional.py +2 -1
  130. data/vendor/pygments-main/pygments/lexers/futhark.py +6 -7
  131. data/vendor/pygments-main/pygments/lexers/gcodelexer.py +3 -3
  132. data/vendor/pygments-main/pygments/lexers/gdscript.py +2 -1
  133. data/vendor/pygments-main/pygments/lexers/go.py +3 -4
  134. data/vendor/pygments-main/pygments/lexers/grammar_notation.py +6 -9
  135. data/vendor/pygments-main/pygments/lexers/graph.py +12 -9
  136. data/vendor/pygments-main/pygments/lexers/graphics.py +13 -17
  137. data/vendor/pygments-main/pygments/lexers/graphql.py +176 -0
  138. data/vendor/pygments-main/pygments/lexers/graphviz.py +2 -3
  139. data/vendor/pygments-main/pygments/lexers/gsql.py +2 -3
  140. data/vendor/pygments-main/pygments/lexers/haskell.py +27 -32
  141. data/vendor/pygments-main/pygments/lexers/haxe.py +3 -5
  142. data/vendor/pygments-main/pygments/lexers/hdl.py +8 -7
  143. data/vendor/pygments-main/pygments/lexers/hexdump.py +3 -3
  144. data/vendor/pygments-main/pygments/lexers/html.py +34 -13
  145. data/vendor/pygments-main/pygments/lexers/idl.py +2 -3
  146. data/vendor/pygments-main/pygments/lexers/igor.py +305 -290
  147. data/vendor/pygments-main/pygments/lexers/inferno.py +2 -3
  148. data/vendor/pygments-main/pygments/lexers/installers.py +7 -9
  149. data/vendor/pygments-main/pygments/lexers/int_fiction.py +150 -162
  150. data/vendor/pygments-main/pygments/lexers/iolang.py +2 -3
  151. data/vendor/pygments-main/pygments/lexers/j.py +2 -3
  152. data/vendor/pygments-main/pygments/lexers/javascript.py +18 -19
  153. data/vendor/pygments-main/pygments/lexers/jmespath.py +2 -1
  154. data/vendor/pygments-main/pygments/lexers/jslt.py +2 -3
  155. data/vendor/pygments-main/pygments/lexers/jsonnet.py +4 -3
  156. data/vendor/pygments-main/pygments/lexers/jsx.py +75 -0
  157. data/vendor/pygments-main/pygments/lexers/julia.py +4 -5
  158. data/vendor/pygments-main/pygments/lexers/jvm.py +98 -116
  159. data/vendor/pygments-main/pygments/lexers/kuin.py +2 -3
  160. data/vendor/pygments-main/pygments/lexers/kusto.py +93 -0
  161. data/vendor/pygments-main/pygments/lexers/ldap.py +155 -0
  162. data/vendor/pygments-main/pygments/lexers/lean.py +241 -0
  163. data/vendor/pygments-main/pygments/lexers/lilypond.py +2 -3
  164. data/vendor/pygments-main/pygments/lexers/lisp.py +373 -65
  165. data/vendor/pygments-main/pygments/lexers/macaulay2.py +53 -4
  166. data/vendor/pygments-main/pygments/lexers/make.py +7 -5
  167. data/vendor/pygments-main/pygments/lexers/markup.py +242 -91
  168. data/vendor/pygments-main/pygments/lexers/math.py +2 -1
  169. data/vendor/pygments-main/pygments/lexers/matlab.py +10 -12
  170. data/vendor/pygments-main/pygments/lexers/maxima.py +2 -3
  171. data/vendor/pygments-main/pygments/lexers/meson.py +2 -3
  172. data/vendor/pygments-main/pygments/lexers/mime.py +4 -4
  173. data/vendor/pygments-main/pygments/lexers/minecraft.py +24 -27
  174. data/vendor/pygments-main/pygments/lexers/mips.py +3 -1
  175. data/vendor/pygments-main/pygments/lexers/ml.py +59 -61
  176. data/vendor/pygments-main/pygments/lexers/modeling.py +19 -22
  177. data/vendor/pygments-main/pygments/lexers/modula2.py +2 -3
  178. data/vendor/pygments-main/pygments/lexers/mojo.py +704 -0
  179. data/vendor/pygments-main/pygments/lexers/monte.py +2 -3
  180. data/vendor/pygments-main/pygments/lexers/mosel.py +3 -3
  181. data/vendor/pygments-main/pygments/lexers/ncl.py +4 -3
  182. data/vendor/pygments-main/pygments/lexers/nimrod.py +7 -8
  183. data/vendor/pygments-main/pygments/lexers/nit.py +2 -3
  184. data/vendor/pygments-main/pygments/lexers/nix.py +40 -31
  185. data/vendor/pygments-main/pygments/lexers/oberon.py +3 -3
  186. data/vendor/pygments-main/pygments/lexers/objective.py +14 -6
  187. data/vendor/pygments-main/pygments/lexers/ooc.py +2 -3
  188. data/vendor/pygments-main/pygments/lexers/openscad.py +96 -0
  189. data/vendor/pygments-main/pygments/lexers/other.py +2 -1
  190. data/vendor/pygments-main/pygments/lexers/parasail.py +2 -3
  191. data/vendor/pygments-main/pygments/lexers/parsers.py +41 -44
  192. data/vendor/pygments-main/pygments/lexers/pascal.py +6 -3
  193. data/vendor/pygments-main/pygments/lexers/pawn.py +5 -5
  194. data/vendor/pygments-main/pygments/lexers/perl.py +3 -3
  195. data/vendor/pygments-main/pygments/lexers/phix.py +2 -3
  196. data/vendor/pygments-main/pygments/lexers/php.py +4 -5
  197. data/vendor/pygments-main/pygments/lexers/pointless.py +2 -3
  198. data/vendor/pygments-main/pygments/lexers/pony.py +3 -3
  199. data/vendor/pygments-main/pygments/lexers/praat.py +2 -3
  200. data/vendor/pygments-main/pygments/lexers/procfile.py +2 -3
  201. data/vendor/pygments-main/pygments/lexers/prolog.py +23 -9
  202. data/vendor/pygments-main/pygments/lexers/promql.py +2 -1
  203. data/vendor/pygments-main/pygments/lexers/prql.py +251 -0
  204. data/vendor/pygments-main/pygments/lexers/ptx.py +119 -0
  205. data/vendor/pygments-main/pygments/lexers/python.py +24 -24
  206. data/vendor/pygments-main/pygments/lexers/q.py +5 -6
  207. data/vendor/pygments-main/pygments/lexers/qlik.py +3 -3
  208. data/vendor/pygments-main/pygments/lexers/qvt.py +5 -3
  209. data/vendor/pygments-main/pygments/lexers/r.py +8 -6
  210. data/vendor/pygments-main/pygments/lexers/rdf.py +19 -13
  211. data/vendor/pygments-main/pygments/lexers/rebol.py +7 -7
  212. data/vendor/pygments-main/pygments/lexers/resource.py +5 -6
  213. data/vendor/pygments-main/pygments/lexers/ride.py +4 -5
  214. data/vendor/pygments-main/pygments/lexers/rita.py +2 -3
  215. data/vendor/pygments-main/pygments/lexers/rnc.py +2 -3
  216. data/vendor/pygments-main/pygments/lexers/roboconf.py +5 -5
  217. data/vendor/pygments-main/pygments/lexers/robotframework.py +2 -3
  218. data/vendor/pygments-main/pygments/lexers/ruby.py +8 -13
  219. data/vendor/pygments-main/pygments/lexers/rust.py +9 -10
  220. data/vendor/pygments-main/pygments/lexers/sas.py +3 -3
  221. data/vendor/pygments-main/pygments/lexers/savi.py +131 -130
  222. data/vendor/pygments-main/pygments/lexers/scdoc.py +13 -7
  223. data/vendor/pygments-main/pygments/lexers/scripting.py +354 -42
  224. data/vendor/pygments-main/pygments/lexers/sgf.py +2 -3
  225. data/vendor/pygments-main/pygments/lexers/shell.py +107 -129
  226. data/vendor/pygments-main/pygments/lexers/sieve.py +3 -3
  227. data/vendor/pygments-main/pygments/lexers/slash.py +4 -5
  228. data/vendor/pygments-main/pygments/lexers/smalltalk.py +3 -5
  229. data/vendor/pygments-main/pygments/lexers/smithy.py +2 -3
  230. data/vendor/pygments-main/pygments/lexers/smv.py +3 -3
  231. data/vendor/pygments-main/pygments/lexers/snobol.py +3 -3
  232. data/vendor/pygments-main/pygments/lexers/solidity.py +3 -3
  233. data/vendor/pygments-main/pygments/lexers/soong.py +78 -0
  234. data/vendor/pygments-main/pygments/lexers/sophia.py +3 -4
  235. data/vendor/pygments-main/pygments/lexers/special.py +8 -3
  236. data/vendor/pygments-main/pygments/lexers/spice.py +15 -16
  237. data/vendor/pygments-main/pygments/lexers/sql.py +24 -18
  238. data/vendor/pygments-main/pygments/lexers/srcinfo.py +3 -3
  239. data/vendor/pygments-main/pygments/lexers/stata.py +2 -3
  240. data/vendor/pygments-main/pygments/lexers/supercollider.py +2 -3
  241. data/vendor/pygments-main/pygments/lexers/tact.py +303 -0
  242. data/vendor/pygments-main/pygments/lexers/tal.py +13 -10
  243. data/vendor/pygments-main/pygments/lexers/tcl.py +2 -3
  244. data/vendor/pygments-main/pygments/lexers/teal.py +2 -3
  245. data/vendor/pygments-main/pygments/lexers/templates.py +131 -72
  246. data/vendor/pygments-main/pygments/lexers/teraterm.py +2 -3
  247. data/vendor/pygments-main/pygments/lexers/testing.py +4 -5
  248. data/vendor/pygments-main/pygments/lexers/text.py +2 -1
  249. data/vendor/pygments-main/pygments/lexers/textedit.py +9 -6
  250. data/vendor/pygments-main/pygments/lexers/textfmts.py +20 -15
  251. data/vendor/pygments-main/pygments/lexers/theorem.py +41 -115
  252. data/vendor/pygments-main/pygments/lexers/thingsdb.py +50 -26
  253. data/vendor/pygments-main/pygments/lexers/tlb.py +3 -1
  254. data/vendor/pygments-main/pygments/lexers/tls.py +54 -0
  255. data/vendor/pygments-main/pygments/lexers/tnt.py +2 -3
  256. data/vendor/pygments-main/pygments/lexers/trafficscript.py +3 -3
  257. data/vendor/pygments-main/pygments/lexers/typoscript.py +6 -7
  258. data/vendor/pygments-main/pygments/lexers/typst.py +104 -0
  259. data/vendor/pygments-main/pygments/lexers/ul4.py +47 -5
  260. data/vendor/pygments-main/pygments/lexers/unicon.py +9 -7
  261. data/vendor/pygments-main/pygments/lexers/urbi.py +3 -3
  262. data/vendor/pygments-main/pygments/lexers/usd.py +6 -11
  263. data/vendor/pygments-main/pygments/lexers/varnish.py +5 -5
  264. data/vendor/pygments-main/pygments/lexers/verification.py +4 -5
  265. data/vendor/pygments-main/pygments/lexers/verifpal.py +65 -0
  266. data/vendor/pygments-main/pygments/lexers/vip.py +150 -0
  267. data/vendor/pygments-main/pygments/lexers/vyper.py +140 -0
  268. data/vendor/pygments-main/pygments/lexers/web.py +2 -1
  269. data/vendor/pygments-main/pygments/lexers/webassembly.py +2 -3
  270. data/vendor/pygments-main/pygments/lexers/webidl.py +2 -3
  271. data/vendor/pygments-main/pygments/lexers/webmisc.py +11 -15
  272. data/vendor/pygments-main/pygments/lexers/wgsl.py +6 -7
  273. data/vendor/pygments-main/pygments/lexers/whiley.py +2 -3
  274. data/vendor/pygments-main/pygments/lexers/wowtoc.py +3 -3
  275. data/vendor/pygments-main/pygments/lexers/wren.py +3 -4
  276. data/vendor/pygments-main/pygments/lexers/x10.py +6 -7
  277. data/vendor/pygments-main/pygments/lexers/xorg.py +2 -1
  278. data/vendor/pygments-main/pygments/lexers/yang.py +2 -3
  279. data/vendor/pygments-main/pygments/lexers/yara.py +69 -0
  280. data/vendor/pygments-main/pygments/lexers/zig.py +2 -1
  281. data/vendor/pygments-main/pygments/modeline.py +4 -4
  282. data/vendor/pygments-main/pygments/plugin.py +3 -19
  283. data/vendor/pygments-main/pygments/regexopt.py +1 -1
  284. data/vendor/pygments-main/pygments/scanner.py +1 -1
  285. data/vendor/pygments-main/pygments/sphinxext.py +38 -8
  286. data/vendor/pygments-main/pygments/style.py +8 -2
  287. data/vendor/pygments-main/pygments/styles/__init__.py +17 -59
  288. data/vendor/pygments-main/pygments/styles/_mapping.py +54 -0
  289. data/vendor/pygments-main/pygments/styles/abap.py +5 -1
  290. data/vendor/pygments-main/pygments/styles/algol.py +5 -1
  291. data/vendor/pygments-main/pygments/styles/algol_nu.py +5 -1
  292. data/vendor/pygments-main/pygments/styles/arduino.py +76 -72
  293. data/vendor/pygments-main/pygments/styles/autumn.py +6 -1
  294. data/vendor/pygments-main/pygments/styles/borland.py +6 -1
  295. data/vendor/pygments-main/pygments/styles/bw.py +6 -1
  296. data/vendor/pygments-main/pygments/styles/coffee.py +80 -0
  297. data/vendor/pygments-main/pygments/styles/colorful.py +6 -1
  298. data/vendor/pygments-main/pygments/styles/default.py +6 -1
  299. data/vendor/pygments-main/pygments/styles/dracula.py +68 -80
  300. data/vendor/pygments-main/pygments/styles/emacs.py +6 -1
  301. data/vendor/pygments-main/pygments/styles/friendly.py +6 -1
  302. data/vendor/pygments-main/pygments/styles/friendly_grayscale.py +6 -1
  303. data/vendor/pygments-main/pygments/styles/fruity.py +7 -1
  304. data/vendor/pygments-main/pygments/styles/gh_dark.py +7 -1
  305. data/vendor/pygments-main/pygments/styles/gruvbox.py +10 -1
  306. data/vendor/pygments-main/pygments/styles/igor.py +6 -1
  307. data/vendor/pygments-main/pygments/styles/inkpot.py +7 -2
  308. data/vendor/pygments-main/pygments/styles/lightbulb.py +110 -0
  309. data/vendor/pygments-main/pygments/styles/lilypond.py +8 -2
  310. data/vendor/pygments-main/pygments/styles/lovelace.py +16 -10
  311. data/vendor/pygments-main/pygments/styles/manni.py +7 -2
  312. data/vendor/pygments-main/pygments/styles/material.py +21 -14
  313. data/vendor/pygments-main/pygments/styles/monokai.py +16 -10
  314. data/vendor/pygments-main/pygments/styles/murphy.py +7 -2
  315. data/vendor/pygments-main/pygments/styles/native.py +10 -5
  316. data/vendor/pygments-main/pygments/styles/nord.py +9 -3
  317. data/vendor/pygments-main/pygments/styles/onedark.py +6 -2
  318. data/vendor/pygments-main/pygments/styles/paraiso_dark.py +7 -2
  319. data/vendor/pygments-main/pygments/styles/paraiso_light.py +7 -2
  320. data/vendor/pygments-main/pygments/styles/pastie.py +7 -1
  321. data/vendor/pygments-main/pygments/styles/perldoc.py +7 -1
  322. data/vendor/pygments-main/pygments/styles/rainbow_dash.py +26 -19
  323. data/vendor/pygments-main/pygments/styles/rrt.py +8 -2
  324. data/vendor/pygments-main/pygments/styles/sas.py +6 -1
  325. data/vendor/pygments-main/pygments/styles/solarized.py +9 -1
  326. data/vendor/pygments-main/pygments/styles/staroffice.py +6 -1
  327. data/vendor/pygments-main/pygments/styles/stata_dark.py +6 -2
  328. data/vendor/pygments-main/pygments/styles/stata_light.py +6 -1
  329. data/vendor/pygments-main/pygments/styles/tango.py +7 -3
  330. data/vendor/pygments-main/pygments/styles/trac.py +7 -1
  331. data/vendor/pygments-main/pygments/styles/vim.py +7 -1
  332. data/vendor/pygments-main/pygments/styles/vs.py +7 -2
  333. data/vendor/pygments-main/pygments/styles/xcode.py +6 -1
  334. data/vendor/pygments-main/pygments/styles/zenburn.py +6 -1
  335. data/vendor/pygments-main/pygments/token.py +2 -1
  336. data/vendor/pygments-main/pygments/unistring.py +5 -5
  337. data/vendor/pygments-main/pygments/util.py +13 -19
  338. data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info}/METADATA +14 -13
  339. data/vendor/pygments-main/pygments-2.18.0.dist-info/RECORD +663 -0
  340. data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info}/WHEEL +1 -2
  341. data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info/licenses}/AUTHORS +13 -0
  342. metadata +39 -15
  343. data/vendor/pygments-main/Pygments-2.15.1.dist-info/RECORD +0 -614
  344. data/vendor/pygments-main/Pygments-2.15.1.dist-info/top_level.txt +0 -1
  345. /data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info}/INSTALLER +0 -0
  346. /data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info}/REQUESTED +0 -0
  347. /data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info}/entry_points.txt +0 -0
  348. /data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info/licenses}/LICENSE +0 -0
@@ -4,12 +4,14 @@
4
4
 
5
5
  A formatter that generates RTF files.
6
6
 
7
- :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
7
+ :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
8
8
  :license: BSD, see LICENSE for details.
9
9
  """
10
10
 
11
+ from collections import OrderedDict
11
12
  from pygments.formatter import Formatter
12
- from pygments.util import get_int_opt, surrogatepair
13
+ from pygments.style import _ansimap
14
+ from pygments.util import get_bool_opt, get_int_opt, get_list_opt, surrogatepair
13
15
 
14
16
 
15
17
  __all__ = ['RtfFormatter']
@@ -42,6 +44,59 @@ class RtfFormatter(Formatter):
42
44
  default is 24 half-points, giving a size 12 font.
43
45
 
44
46
  .. versionadded:: 2.0
47
+
48
+ `linenos`
49
+ Turn on line numbering (default: ``False``).
50
+
51
+ .. versionadded:: 2.18
52
+
53
+ `lineno_fontsize`
54
+ Font size for line numbers. Size is specified in half points
55
+ (default: `fontsize`).
56
+
57
+ .. versionadded:: 2.18
58
+
59
+ `lineno_padding`
60
+ Number of spaces between the (inline) line numbers and the
61
+ source code (default: ``2``).
62
+
63
+ .. versionadded:: 2.18
64
+
65
+ `linenostart`
66
+ The line number for the first line (default: ``1``).
67
+
68
+ .. versionadded:: 2.18
69
+
70
+ `linenostep`
71
+ If set to a number n > 1, only every nth line number is printed.
72
+
73
+ .. versionadded:: 2.18
74
+
75
+ `lineno_color`
76
+ Color for line numbers specified as a hex triplet, e.g. ``'5e5e5e'``.
77
+ Defaults to the style's line number color if it is a hex triplet,
78
+ otherwise ansi bright black.
79
+
80
+ .. versionadded:: 2.18
81
+
82
+ `hl_lines`
83
+ Specify a list of lines to be highlighted, as line numbers separated by
84
+ spaces, e.g. ``'3 7 8'``. The line numbers are relative to the input
85
+ (i.e. the first line is line 1) unless `hl_linenostart` is set.
86
+
87
+ .. versionadded:: 2.18
88
+
89
+ `hl_color`
90
+ Color for highlighting the lines specified in `hl_lines`, specified as
91
+ a hex triplet (default: style's `highlight_color`).
92
+
93
+ .. versionadded:: 2.18
94
+
95
+ `hl_linenostart`
96
+ If set to ``True`` line numbers in `hl_lines` are specified
97
+ relative to `linenostart` (default ``False``).
98
+
99
+ .. versionadded:: 2.18
45
100
  """
46
101
  name = 'RTF'
47
102
  aliases = ['rtf']
@@ -62,6 +117,40 @@ class RtfFormatter(Formatter):
62
117
  Formatter.__init__(self, **options)
63
118
  self.fontface = options.get('fontface') or ''
64
119
  self.fontsize = get_int_opt(options, 'fontsize', 0)
120
+ self.linenos = get_bool_opt(options, 'linenos', False)
121
+ self.lineno_fontsize = get_int_opt(options, 'lineno_fontsize',
122
+ self.fontsize)
123
+ self.lineno_padding = get_int_opt(options, 'lineno_padding', 2)
124
+ self.linenostart = abs(get_int_opt(options, 'linenostart', 1))
125
+ self.linenostep = abs(get_int_opt(options, 'linenostep', 1))
126
+ self.hl_linenostart = get_bool_opt(options, 'hl_linenostart', False)
127
+
128
+ self.hl_color = options.get('hl_color', '')
129
+ if not self.hl_color:
130
+ self.hl_color = self.style.highlight_color
131
+
132
+ self.hl_lines = []
133
+ for lineno in get_list_opt(options, 'hl_lines', []):
134
+ try:
135
+ lineno = int(lineno)
136
+ if self.hl_linenostart:
137
+ lineno = lineno - self.linenostart + 1
138
+ self.hl_lines.append(lineno)
139
+ except ValueError:
140
+ pass
141
+
142
+ self.lineno_color = options.get('lineno_color', '')
143
+ if not self.lineno_color:
144
+ if self.style.line_number_color == 'inherit':
145
+ # style color is the css value 'inherit'
146
+ # default to ansi bright-black
147
+ self.lineno_color = _ansimap['ansibrightblack']
148
+ else:
149
+ # style color is assumed to be a hex triplet as other
150
+ # colors in pygments/style.py
151
+ self.lineno_color = self.style.line_number_color
152
+
153
+ self.color_mapping = self._create_color_mapping()
65
154
 
66
155
  def _escape(self, text):
67
156
  return text.replace('\\', '\\\\') \
@@ -90,43 +179,145 @@ class RtfFormatter(Formatter):
90
179
  # Force surrogate pairs
91
180
  buf.append('{\\u%d}{\\u%d}' % surrogatepair(cn))
92
181
 
93
- return ''.join(buf).replace('\n', '\\par\n')
182
+ return ''.join(buf).replace('\n', '\\par')
94
183
 
95
- def format_unencoded(self, tokensource, outfile):
96
- # rtf 1.8 header
97
- outfile.write('{\\rtf1\\ansi\\uc0\\deff0'
98
- '{\\fonttbl{\\f0\\fmodern\\fprq1\\fcharset0%s;}}'
99
- '{\\colortbl;' % (self.fontface and
100
- ' ' + self._escape(self.fontface) or
101
- ''))
102
-
103
- # convert colors and save them in a mapping to access them later.
104
- color_mapping = {}
184
+ @staticmethod
185
+ def hex_to_rtf_color(hex_color):
186
+ if hex_color[0] == "#":
187
+ hex_color = hex_color[1:]
188
+
189
+ return '\\red%d\\green%d\\blue%d;' % (
190
+ int(hex_color[0:2], 16),
191
+ int(hex_color[2:4], 16),
192
+ int(hex_color[4:6], 16)
193
+ )
194
+
195
+ def _split_tokens_on_newlines(self, tokensource):
196
+ """
197
+ Split tokens containing newline characters into multiple token
198
+ each representing a line of the input file. Needed for numbering
199
+ lines of e.g. multiline comments.
200
+ """
201
+ for ttype, value in tokensource:
202
+ if value == '\n':
203
+ yield (ttype, value)
204
+ elif "\n" in value:
205
+ lines = value.split("\n")
206
+ for line in lines[:-1]:
207
+ yield (ttype, line+"\n")
208
+ if lines[-1]:
209
+ yield (ttype, lines[-1])
210
+ else:
211
+ yield (ttype, value)
212
+
213
+ def _create_color_mapping(self):
214
+ """
215
+ Create a mapping of style hex colors to index/offset in
216
+ the RTF color table.
217
+ """
218
+ color_mapping = OrderedDict()
105
219
  offset = 1
220
+
221
+ if self.linenos:
222
+ color_mapping[self.lineno_color] = offset
223
+ offset += 1
224
+
225
+ if self.hl_lines:
226
+ color_mapping[self.hl_color] = offset
227
+ offset += 1
228
+
106
229
  for _, style in self.style:
107
230
  for color in style['color'], style['bgcolor'], style['border']:
108
231
  if color and color not in color_mapping:
109
232
  color_mapping[color] = offset
110
- outfile.write('\\red%d\\green%d\\blue%d;' % (
111
- int(color[0:2], 16),
112
- int(color[2:4], 16),
113
- int(color[4:6], 16)
114
- ))
115
233
  offset += 1
116
- outfile.write('}\\f0 ')
234
+
235
+ return color_mapping
236
+
237
+ @property
238
+ def _lineno_template(self):
239
+ if self.lineno_fontsize != self.fontsize:
240
+ return '{{\\fs{} \\cf{} %s{}}}'.format(self.lineno_fontsize,
241
+ self.color_mapping[self.lineno_color],
242
+ " " * self.lineno_padding)
243
+
244
+ return '{{\\cf{} %s{}}}'.format(self.color_mapping[self.lineno_color],
245
+ " " * self.lineno_padding)
246
+
247
+ @property
248
+ def _hl_open_str(self):
249
+ return rf'{{\highlight{self.color_mapping[self.hl_color]} '
250
+
251
+ @property
252
+ def _rtf_header(self):
253
+ lines = []
254
+ # rtf 1.8 header
255
+ lines.append('{\\rtf1\\ansi\\uc0\\deff0'
256
+ '{\\fonttbl{\\f0\\fmodern\\fprq1\\fcharset0%s;}}'
257
+ % (self.fontface and ' '
258
+ + self._escape(self.fontface) or ''))
259
+
260
+ # color table
261
+ lines.append('{\\colortbl;')
262
+ for color, _ in self.color_mapping.items():
263
+ lines.append(self.hex_to_rtf_color(color))
264
+ lines.append('}')
265
+
266
+ # font and fontsize
267
+ lines.append('\\f0\\sa0')
117
268
  if self.fontsize:
118
- outfile.write('\\fs%d' % self.fontsize)
269
+ lines.append('\\fs%d' % self.fontsize)
270
+
271
+ # ensure Libre Office Writer imports and renders consecutive
272
+ # space characters the same width, needed for line numbering.
273
+ # https://bugs.documentfoundation.org/show_bug.cgi?id=144050
274
+ lines.append('\\dntblnsbdb')
275
+
276
+ return lines
277
+
278
+ def format_unencoded(self, tokensource, outfile):
279
+ for line in self._rtf_header:
280
+ outfile.write(line + "\n")
281
+
282
+ tokensource = self._split_tokens_on_newlines(tokensource)
283
+
284
+ # first pass of tokens to count lines, needed for line numbering
285
+ if self.linenos:
286
+ line_count = 0
287
+ tokens = [] # for copying the token source generator
288
+ for ttype, value in tokensource:
289
+ tokens.append((ttype, value))
290
+ if value.endswith("\n"):
291
+ line_count += 1
292
+
293
+ # width of line number strings (for padding with spaces)
294
+ linenos_width = len(str(line_count+self.linenostart-1))
295
+
296
+ tokensource = tokens
119
297
 
120
298
  # highlight stream
299
+ lineno = 1
300
+ start_new_line = True
121
301
  for ttype, value in tokensource:
302
+ if start_new_line and lineno in self.hl_lines:
303
+ outfile.write(self._hl_open_str)
304
+
305
+ if start_new_line and self.linenos:
306
+ if (lineno-self.linenostart+1)%self.linenostep == 0:
307
+ current_lineno = lineno + self.linenostart - 1
308
+ lineno_str = str(current_lineno).rjust(linenos_width)
309
+ else:
310
+ lineno_str = "".rjust(linenos_width)
311
+ outfile.write(self._lineno_template % lineno_str)
312
+
122
313
  while not self.style.styles_token(ttype) and ttype.parent:
123
314
  ttype = ttype.parent
124
315
  style = self.style.style_for_token(ttype)
125
316
  buf = []
126
317
  if style['bgcolor']:
127
- buf.append('\\cb%d' % color_mapping[style['bgcolor']])
318
+ buf.append('\\cb%d' % self.color_mapping[style['bgcolor']])
128
319
  if style['color']:
129
- buf.append('\\cf%d' % color_mapping[style['color']])
320
+ buf.append('\\cf%d' % self.color_mapping[style['color']])
130
321
  if style['bold']:
131
322
  buf.append('\\b')
132
323
  if style['italic']:
@@ -135,12 +326,24 @@ class RtfFormatter(Formatter):
135
326
  buf.append('\\ul')
136
327
  if style['border']:
137
328
  buf.append('\\chbrdr\\chcfpat%d' %
138
- color_mapping[style['border']])
329
+ self.color_mapping[style['border']])
139
330
  start = ''.join(buf)
140
331
  if start:
141
- outfile.write('{%s ' % start)
332
+ outfile.write(f'{{{start} ')
142
333
  outfile.write(self._escape_text(value))
143
334
  if start:
144
335
  outfile.write('}')
336
+ start_new_line = False
337
+
338
+ # complete line of input
339
+ if value.endswith("\n"):
340
+ # close line highlighting
341
+ if lineno in self.hl_lines:
342
+ outfile.write('}')
343
+ # newline in RTF file after closing }
344
+ outfile.write("\n")
345
+
346
+ start_new_line = True
347
+ lineno += 1
145
348
 
146
- outfile.write('}')
349
+ outfile.write('}\n')
@@ -4,7 +4,7 @@
4
4
 
5
5
  Formatter for SVG output.
6
6
 
7
- :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
7
+ :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
8
8
  :license: BSD, see LICENSE for details.
9
9
  """
10
10
 
@@ -60,11 +60,11 @@ class SvgFormatter(Formatter):
60
60
 
61
61
  `linenostep`
62
62
  If set to a number n > 1, only every nth line number is printed.
63
-
63
+
64
64
  `linenowidth`
65
65
  Maximum width devoted to line numbers (default: ``3*ystep``, sufficient
66
- for up to 4-digit line numbers. Increase width for longer code blocks).
67
-
66
+ for up to 4-digit line numbers. Increase width for longer code blocks).
67
+
68
68
  `xoffset`
69
69
  Starting offset in X direction, defaults to ``0``.
70
70
 
@@ -97,10 +97,11 @@ class SvgFormatter(Formatter):
97
97
  self.fontsize = options.get('fontsize', '14px')
98
98
  self.xoffset = get_int_opt(options, 'xoffset', 0)
99
99
  fs = self.fontsize.strip()
100
- if fs.endswith('px'): fs = fs[:-2].strip()
100
+ if fs.endswith('px'):
101
+ fs = fs[:-2].strip()
101
102
  try:
102
103
  int_fs = int(fs)
103
- except:
104
+ except ValueError:
104
105
  int_fs = 20
105
106
  self.yoffset = get_int_opt(options, 'yoffset', int_fs)
106
107
  self.ystep = get_int_opt(options, 'ystep', int_fs + 5)
@@ -122,30 +123,27 @@ class SvgFormatter(Formatter):
122
123
  y = self.yoffset
123
124
  if not self.nowrap:
124
125
  if self.encoding:
125
- outfile.write('<?xml version="1.0" encoding="%s"?>\n' %
126
- self.encoding)
126
+ outfile.write(f'<?xml version="1.0" encoding="{self.encoding}"?>\n')
127
127
  else:
128
128
  outfile.write('<?xml version="1.0"?>\n')
129
129
  outfile.write('<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" '
130
130
  '"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/'
131
131
  'svg10.dtd">\n')
132
132
  outfile.write('<svg xmlns="http://www.w3.org/2000/svg">\n')
133
- outfile.write('<g font-family="%s" font-size="%s">\n' %
134
- (self.fontfamily, self.fontsize))
135
-
136
- counter = self.linenostart
133
+ outfile.write(f'<g font-family="{self.fontfamily}" font-size="{self.fontsize}">\n')
134
+
135
+ counter = self.linenostart
137
136
  counter_step = self.linenostep
138
137
  counter_style = self._get_style(Comment)
139
138
  line_x = x
140
-
139
+
141
140
  if self.linenos:
142
141
  if counter % counter_step == 0:
143
- outfile.write('<text x="%s" y="%s" %s text-anchor="end">%s</text>' %
144
- (x+self.linenowidth,y,counter_style,counter))
142
+ outfile.write(f'<text x="{x+self.linenowidth}" y="{y}" {counter_style} text-anchor="end">{counter}</text>')
145
143
  line_x += self.linenowidth + self.ystep
146
144
  counter += 1
147
145
 
148
- outfile.write('<text x="%s" y="%s" xml:space="preserve">' % (line_x, y))
146
+ outfile.write(f'<text x="{line_x}" y="{y}" xml:space="preserve">')
149
147
  for ttype, value in tokensource:
150
148
  style = self._get_style(ttype)
151
149
  tspan = style and '<tspan' + style + '>' or ''
@@ -159,11 +157,10 @@ class SvgFormatter(Formatter):
159
157
  y += self.ystep
160
158
  outfile.write('</text>\n')
161
159
  if self.linenos and counter % counter_step == 0:
162
- outfile.write('<text x="%s" y="%s" text-anchor="end" %s>%s</text>' %
163
- (x+self.linenowidth,y,counter_style,counter))
164
-
160
+ outfile.write(f'<text x="{x+self.linenowidth}" y="{y}" text-anchor="end" {counter_style}>{counter}</text>')
161
+
165
162
  counter += 1
166
- outfile.write('<text x="%s" y="%s" ' 'xml:space="preserve">' % (line_x,y))
163
+ outfile.write(f'<text x="{line_x}" y="{y}" ' 'xml:space="preserve">')
167
164
  outfile.write(tspan + parts[-1] + tspanend)
168
165
  outfile.write('</text>')
169
166
 
@@ -4,7 +4,7 @@
4
4
 
5
5
  Formatter for terminal output with ANSI sequences.
6
6
 
7
- :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
7
+ :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
8
8
  :license: BSD, see LICENSE for details.
9
9
  """
10
10
 
@@ -10,7 +10,7 @@
10
10
 
11
11
  Formatter version 1.
12
12
 
13
- :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
13
+ :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
14
14
  :license: BSD, see LICENSE for details.
15
15
  """
16
16
 
@@ -4,7 +4,7 @@
4
4
 
5
5
  Base lexer classes.
6
6
 
7
- :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
7
+ :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
8
8
  :license: BSD, see LICENSE for details.
9
9
  """
10
10
 
@@ -67,10 +67,17 @@ class Lexer(metaclass=LexerMeta):
67
67
  :no-value:
68
68
  .. autoattribute:: priority
69
69
 
70
- Lexers included in Pygments should have an additional attribute:
70
+ Lexers included in Pygments should have two additional attributes:
71
71
 
72
72
  .. autoattribute:: url
73
73
  :no-value:
74
+ .. autoattribute:: version_added
75
+ :no-value:
76
+
77
+ Lexers included in Pygments may have additional attributes:
78
+
79
+ .. autoattribute:: _example
80
+ :no-value:
74
81
 
75
82
  You can pass options to the constructor. The basic options recognized
76
83
  by all lexers and processed by the base `Lexer` class are:
@@ -125,9 +132,16 @@ class Lexer(metaclass=LexerMeta):
125
132
  priority = 0
126
133
 
127
134
  #: URL of the language specification/definition. Used in the Pygments
128
- #: documentation.
135
+ #: documentation. Set to an empty string to disable.
129
136
  url = None
130
137
 
138
+ #: Version of Pygments in which the lexer was added.
139
+ version_added = None
140
+
141
+ #: Example file name. Relative to the ``tests/examplefiles`` directory.
142
+ #: This is used by the documentation generator to show an example.
143
+ _example = None
144
+
131
145
  def __init__(self, **options):
132
146
  """
133
147
  This constructor takes arbitrary options as keyword arguments.
@@ -160,10 +174,9 @@ class Lexer(metaclass=LexerMeta):
160
174
 
161
175
  def __repr__(self):
162
176
  if self.options:
163
- return '<pygments.lexers.%s with %r>' % (self.__class__.__name__,
164
- self.options)
177
+ return f'<pygments.lexers.{self.__class__.__name__} with {self.options!r}>'
165
178
  else:
166
- return '<pygments.lexers.%s>' % self.__class__.__name__
179
+ return f'<pygments.lexers.{self.__class__.__name__}>'
167
180
 
168
181
  def add_filter(self, filter_, **options):
169
182
  """
@@ -190,20 +203,9 @@ class Lexer(metaclass=LexerMeta):
190
203
  it's the same as if the return values was ``0.0``.
191
204
  """
192
205
 
193
- def get_tokens(self, text, unfiltered=False):
194
- """
195
- This method is the basic interface of a lexer. It is called by
196
- the `highlight()` function. It must process the text and return an
197
- iterable of ``(tokentype, value)`` pairs from `text`.
198
-
199
- Normally, you don't need to override this method. The default
200
- implementation processes the options recognized by all lexers
201
- (`stripnl`, `stripall` and so on), and then yields all tokens
202
- from `get_tokens_unprocessed()`, with the ``index`` dropped.
206
+ def _preprocess_lexer_input(self, text):
207
+ """Apply preprocessing such as decoding the input, removing BOM and normalizing newlines."""
203
208
 
204
- If `unfiltered` is set to `True`, the filtering mechanism is
205
- bypassed even if filters are defined.
206
- """
207
209
  if not isinstance(text, str):
208
210
  if self.encoding == 'guess':
209
211
  text, _ = guess_decode(text)
@@ -246,6 +248,24 @@ class Lexer(metaclass=LexerMeta):
246
248
  if self.ensurenl and not text.endswith('\n'):
247
249
  text += '\n'
248
250
 
251
+ return text
252
+
253
+ def get_tokens(self, text, unfiltered=False):
254
+ """
255
+ This method is the basic interface of a lexer. It is called by
256
+ the `highlight()` function. It must process the text and return an
257
+ iterable of ``(tokentype, value)`` pairs from `text`.
258
+
259
+ Normally, you don't need to override this method. The default
260
+ implementation processes the options recognized by all lexers
261
+ (`stripnl`, `stripall` and so on), and then yields all tokens
262
+ from `get_tokens_unprocessed()`, with the ``index`` dropped.
263
+
264
+ If `unfiltered` is set to `True`, the filtering mechanism is
265
+ bypassed even if filters are defined.
266
+ """
267
+ text = self._preprocess_lexer_input(text)
268
+
249
269
  def streamer():
250
270
  for _, t, v in self.get_tokens_unprocessed(text):
251
271
  yield t, v
@@ -490,7 +510,7 @@ class RegexLexerMeta(LexerMeta):
490
510
  def _process_token(cls, token):
491
511
  """Preprocess the token component of a token definition."""
492
512
  assert type(token) is _TokenType or callable(token), \
493
- 'token type must be simple type or callable, not %r' % (token,)
513
+ f'token type must be simple type or callable, not {token!r}'
494
514
  return token
495
515
 
496
516
  def _process_new_state(cls, new_state, unprocessed, processed):
@@ -506,14 +526,14 @@ class RegexLexerMeta(LexerMeta):
506
526
  elif new_state[:5] == '#pop:':
507
527
  return -int(new_state[5:])
508
528
  else:
509
- assert False, 'unknown new state %r' % new_state
529
+ assert False, f'unknown new state {new_state!r}'
510
530
  elif isinstance(new_state, combined):
511
531
  # combine a new state from existing ones
512
532
  tmp_state = '_tmp_%d' % cls._tmpname
513
533
  cls._tmpname += 1
514
534
  itokens = []
515
535
  for istate in new_state:
516
- assert istate != new_state, 'circular state ref %r' % istate
536
+ assert istate != new_state, f'circular state ref {istate!r}'
517
537
  itokens.extend(cls._process_state(unprocessed,
518
538
  processed, istate))
519
539
  processed[tmp_state] = itokens
@@ -526,12 +546,12 @@ class RegexLexerMeta(LexerMeta):
526
546
  'unknown new state ' + istate
527
547
  return new_state
528
548
  else:
529
- assert False, 'unknown new state def %r' % new_state
549
+ assert False, f'unknown new state def {new_state!r}'
530
550
 
531
551
  def _process_state(cls, unprocessed, processed, state):
532
552
  """Preprocess a single state definition."""
533
- assert type(state) is str, "wrong state name %r" % state
534
- assert state[0] != '#', "invalid state name %r" % state
553
+ assert isinstance(state, str), f"wrong state name {state!r}"
554
+ assert state[0] != '#', f"invalid state name {state!r}"
535
555
  if state in processed:
536
556
  return processed[state]
537
557
  tokens = processed[state] = []
@@ -539,7 +559,7 @@ class RegexLexerMeta(LexerMeta):
539
559
  for tdef in unprocessed[state]:
540
560
  if isinstance(tdef, include):
541
561
  # it's a state reference
542
- assert tdef != state, "circular state reference %r" % state
562
+ assert tdef != state, f"circular state reference {state!r}"
543
563
  tokens.extend(cls._process_state(unprocessed, processed,
544
564
  str(tdef)))
545
565
  continue
@@ -553,13 +573,12 @@ class RegexLexerMeta(LexerMeta):
553
573
  tokens.append((re.compile('').match, None, new_state))
554
574
  continue
555
575
 
556
- assert type(tdef) is tuple, "wrong rule def %r" % tdef
576
+ assert type(tdef) is tuple, f"wrong rule def {tdef!r}"
557
577
 
558
578
  try:
559
579
  rex = cls._process_regex(tdef[0], rflags, state)
560
580
  except Exception as err:
561
- raise ValueError("uncompilable regex %r in state %r of %r: %s" %
562
- (tdef[0], state, cls, err)) from err
581
+ raise ValueError(f"uncompilable regex {tdef[0]!r} in state {state!r} of {cls!r}: {err}") from err
563
582
 
564
583
  token = cls._process_token(tdef[1])
565
584
 
@@ -720,7 +739,7 @@ class RegexLexer(Lexer, metaclass=RegexLexerMeta):
720
739
  elif new_state == '#push':
721
740
  statestack.append(statestack[-1])
722
741
  else:
723
- assert False, "wrong state def: %r" % new_state
742
+ assert False, f"wrong state def: {new_state!r}"
724
743
  statetokens = tokendefs[statestack[-1]]
725
744
  break
726
745
  else:
@@ -752,8 +771,7 @@ class LexerContext:
752
771
  self.stack = stack or ['root']
753
772
 
754
773
  def __repr__(self):
755
- return 'LexerContext(%r, %r, %r)' % (
756
- self.text, self.pos, self.stack)
774
+ return f'LexerContext({self.text!r}, {self.pos!r}, {self.stack!r})'
757
775
 
758
776
 
759
777
  class ExtendedRegexLexer(RegexLexer):
@@ -808,7 +826,7 @@ class ExtendedRegexLexer(RegexLexer):
808
826
  elif new_state == '#push':
809
827
  ctx.stack.append(ctx.stack[-1])
810
828
  else:
811
- assert False, "wrong state def: %r" % new_state
829
+ assert False, f"wrong state def: {new_state!r}"
812
830
  statetokens = tokendefs[ctx.stack[-1]]
813
831
  break
814
832
  else: