pygments.rb 2.4.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +8 -8
- data/.github/workflows/release.yml +5 -5
- data/.gitignore +2 -0
- data/.rubocop.yml +1 -1
- data/CHANGELOG.adoc +9 -1
- data/README.adoc +2 -2
- data/lib/pygments/version.rb +1 -1
- data/pygments.rb.gemspec +2 -2
- data/vendor/pygments-main/bin/pygmentize +1 -1
- data/vendor/pygments-main/pygments/__init__.py +2 -2
- data/vendor/pygments-main/pygments/__main__.py +1 -1
- data/vendor/pygments-main/pygments/cmdline.py +12 -12
- data/vendor/pygments-main/pygments/console.py +5 -5
- data/vendor/pygments-main/pygments/filter.py +2 -3
- data/vendor/pygments-main/pygments/filters/__init__.py +4 -4
- data/vendor/pygments-main/pygments/formatter.py +6 -1
- data/vendor/pygments-main/pygments/formatters/__init__.py +7 -8
- data/vendor/pygments-main/pygments/formatters/_mapping.py +0 -0
- data/vendor/pygments-main/pygments/formatters/bbcode.py +2 -2
- data/vendor/pygments-main/pygments/formatters/groff.py +3 -3
- data/vendor/pygments-main/pygments/formatters/html.py +38 -40
- data/vendor/pygments-main/pygments/formatters/img.py +51 -11
- data/vendor/pygments-main/pygments/formatters/irc.py +1 -1
- data/vendor/pygments-main/pygments/formatters/latex.py +24 -27
- data/vendor/pygments-main/pygments/formatters/other.py +3 -4
- data/vendor/pygments-main/pygments/formatters/pangomarkup.py +2 -2
- data/vendor/pygments-main/pygments/formatters/rtf.py +228 -25
- data/vendor/pygments-main/pygments/formatters/svg.py +17 -20
- data/vendor/pygments-main/pygments/formatters/terminal.py +1 -1
- data/vendor/pygments-main/pygments/formatters/terminal256.py +1 -1
- data/vendor/pygments-main/pygments/lexer.py +51 -33
- data/vendor/pygments-main/pygments/lexers/__init__.py +12 -12
- data/vendor/pygments-main/pygments/lexers/_ada_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_asy_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_cl_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_cocoa_builtins.py +2 -2
- data/vendor/pygments-main/pygments/lexers/_csound_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_css_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_julia_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_lasso_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_lilypond_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_lua_builtins.py +5 -5
- data/vendor/pygments-main/pygments/lexers/_luau_builtins.py +62 -0
- data/vendor/pygments-main/pygments/lexers/_mapping.py +41 -11
- data/vendor/pygments-main/pygments/lexers/_mql_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_mysql_builtins.py +3 -3
- data/vendor/pygments-main/pygments/lexers/_openedge_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_php_builtins.py +3 -3
- data/vendor/pygments-main/pygments/lexers/_postgres_builtins.py +5 -5
- data/vendor/pygments-main/pygments/lexers/_qlik_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_scheme_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_scilab_builtins.py +4 -4
- data/vendor/pygments-main/pygments/lexers/_sourcemod_builtins.py +3 -3
- data/vendor/pygments-main/pygments/lexers/_stan_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_stata_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_tsql_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_usd_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_vbscript_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/_vim_builtins.py +1 -1
- data/vendor/pygments-main/pygments/lexers/actionscript.py +6 -8
- data/vendor/pygments-main/pygments/lexers/ada.py +3 -3
- data/vendor/pygments-main/pygments/lexers/agile.py +3 -1
- data/vendor/pygments-main/pygments/lexers/algebra.py +9 -13
- data/vendor/pygments-main/pygments/lexers/ambient.py +2 -3
- data/vendor/pygments-main/pygments/lexers/amdgpu.py +3 -3
- data/vendor/pygments-main/pygments/lexers/ampl.py +2 -3
- data/vendor/pygments-main/pygments/lexers/apdlexer.py +4 -3
- data/vendor/pygments-main/pygments/lexers/apl.py +2 -3
- data/vendor/pygments-main/pygments/lexers/archetype.py +8 -12
- data/vendor/pygments-main/pygments/lexers/arrow.py +2 -3
- data/vendor/pygments-main/pygments/lexers/arturo.py +2 -3
- data/vendor/pygments-main/pygments/lexers/asc.py +4 -4
- data/vendor/pygments-main/pygments/lexers/asm.py +28 -15
- data/vendor/pygments-main/pygments/lexers/asn1.py +178 -0
- data/vendor/pygments-main/pygments/lexers/automation.py +6 -8
- data/vendor/pygments-main/pygments/lexers/bare.py +2 -3
- data/vendor/pygments-main/pygments/lexers/basic.py +48 -57
- data/vendor/pygments-main/pygments/lexers/bdd.py +3 -4
- data/vendor/pygments-main/pygments/lexers/berry.py +4 -4
- data/vendor/pygments-main/pygments/lexers/bibtex.py +5 -5
- data/vendor/pygments-main/pygments/lexers/blueprint.py +173 -0
- data/vendor/pygments-main/pygments/lexers/boa.py +4 -4
- data/vendor/pygments-main/pygments/lexers/bqn.py +109 -0
- data/vendor/pygments-main/pygments/lexers/business.py +21 -22
- data/vendor/pygments-main/pygments/lexers/c_cpp.py +9 -4
- data/vendor/pygments-main/pygments/lexers/c_like.py +99 -27
- data/vendor/pygments-main/pygments/lexers/capnproto.py +2 -3
- data/vendor/pygments-main/pygments/lexers/carbon.py +10 -11
- data/vendor/pygments-main/pygments/lexers/cddl.py +8 -9
- data/vendor/pygments-main/pygments/lexers/chapel.py +2 -3
- data/vendor/pygments-main/pygments/lexers/clean.py +2 -1
- data/vendor/pygments-main/pygments/lexers/comal.py +2 -1
- data/vendor/pygments-main/pygments/lexers/compiled.py +2 -1
- data/vendor/pygments-main/pygments/lexers/configs.py +342 -108
- data/vendor/pygments-main/pygments/lexers/console.py +5 -5
- data/vendor/pygments-main/pygments/lexers/cplint.py +2 -3
- data/vendor/pygments-main/pygments/lexers/crystal.py +4 -5
- data/vendor/pygments-main/pygments/lexers/csound.py +6 -8
- data/vendor/pygments-main/pygments/lexers/css.py +5 -5
- data/vendor/pygments-main/pygments/lexers/d.py +4 -3
- data/vendor/pygments-main/pygments/lexers/dalvik.py +2 -3
- data/vendor/pygments-main/pygments/lexers/data.py +8 -12
- data/vendor/pygments-main/pygments/lexers/dax.py +2 -3
- data/vendor/pygments-main/pygments/lexers/devicetree.py +2 -3
- data/vendor/pygments-main/pygments/lexers/diff.py +7 -6
- data/vendor/pygments-main/pygments/lexers/dns.py +109 -0
- data/vendor/pygments-main/pygments/lexers/dotnet.py +19 -14
- data/vendor/pygments-main/pygments/lexers/dsls.py +24 -35
- data/vendor/pygments-main/pygments/lexers/dylan.py +8 -16
- data/vendor/pygments-main/pygments/lexers/ecl.py +2 -3
- data/vendor/pygments-main/pygments/lexers/eiffel.py +3 -4
- data/vendor/pygments-main/pygments/lexers/elm.py +3 -4
- data/vendor/pygments-main/pygments/lexers/elpi.py +28 -21
- data/vendor/pygments-main/pygments/lexers/email.py +3 -3
- data/vendor/pygments-main/pygments/lexers/erlang.py +22 -24
- data/vendor/pygments-main/pygments/lexers/esoteric.py +11 -12
- data/vendor/pygments-main/pygments/lexers/ezhil.py +2 -3
- data/vendor/pygments-main/pygments/lexers/factor.py +2 -3
- data/vendor/pygments-main/pygments/lexers/fantom.py +3 -3
- data/vendor/pygments-main/pygments/lexers/felix.py +9 -10
- data/vendor/pygments-main/pygments/lexers/fift.py +2 -1
- data/vendor/pygments-main/pygments/lexers/floscript.py +2 -3
- data/vendor/pygments-main/pygments/lexers/forth.py +2 -3
- data/vendor/pygments-main/pygments/lexers/fortran.py +6 -7
- data/vendor/pygments-main/pygments/lexers/foxpro.py +3 -3
- data/vendor/pygments-main/pygments/lexers/freefem.py +2 -3
- data/vendor/pygments-main/pygments/lexers/func.py +3 -1
- data/vendor/pygments-main/pygments/lexers/functional.py +2 -1
- data/vendor/pygments-main/pygments/lexers/futhark.py +6 -7
- data/vendor/pygments-main/pygments/lexers/gcodelexer.py +3 -3
- data/vendor/pygments-main/pygments/lexers/gdscript.py +2 -1
- data/vendor/pygments-main/pygments/lexers/go.py +3 -4
- data/vendor/pygments-main/pygments/lexers/grammar_notation.py +6 -9
- data/vendor/pygments-main/pygments/lexers/graph.py +12 -9
- data/vendor/pygments-main/pygments/lexers/graphics.py +13 -17
- data/vendor/pygments-main/pygments/lexers/graphql.py +176 -0
- data/vendor/pygments-main/pygments/lexers/graphviz.py +2 -3
- data/vendor/pygments-main/pygments/lexers/gsql.py +2 -3
- data/vendor/pygments-main/pygments/lexers/haskell.py +27 -32
- data/vendor/pygments-main/pygments/lexers/haxe.py +3 -5
- data/vendor/pygments-main/pygments/lexers/hdl.py +8 -7
- data/vendor/pygments-main/pygments/lexers/hexdump.py +3 -3
- data/vendor/pygments-main/pygments/lexers/html.py +34 -13
- data/vendor/pygments-main/pygments/lexers/idl.py +2 -3
- data/vendor/pygments-main/pygments/lexers/igor.py +305 -290
- data/vendor/pygments-main/pygments/lexers/inferno.py +2 -3
- data/vendor/pygments-main/pygments/lexers/installers.py +7 -9
- data/vendor/pygments-main/pygments/lexers/int_fiction.py +150 -162
- data/vendor/pygments-main/pygments/lexers/iolang.py +2 -3
- data/vendor/pygments-main/pygments/lexers/j.py +2 -3
- data/vendor/pygments-main/pygments/lexers/javascript.py +18 -19
- data/vendor/pygments-main/pygments/lexers/jmespath.py +2 -1
- data/vendor/pygments-main/pygments/lexers/jslt.py +2 -3
- data/vendor/pygments-main/pygments/lexers/jsonnet.py +4 -3
- data/vendor/pygments-main/pygments/lexers/jsx.py +75 -0
- data/vendor/pygments-main/pygments/lexers/julia.py +4 -5
- data/vendor/pygments-main/pygments/lexers/jvm.py +98 -116
- data/vendor/pygments-main/pygments/lexers/kuin.py +2 -3
- data/vendor/pygments-main/pygments/lexers/kusto.py +93 -0
- data/vendor/pygments-main/pygments/lexers/ldap.py +155 -0
- data/vendor/pygments-main/pygments/lexers/lean.py +241 -0
- data/vendor/pygments-main/pygments/lexers/lilypond.py +2 -3
- data/vendor/pygments-main/pygments/lexers/lisp.py +373 -65
- data/vendor/pygments-main/pygments/lexers/macaulay2.py +53 -4
- data/vendor/pygments-main/pygments/lexers/make.py +7 -5
- data/vendor/pygments-main/pygments/lexers/markup.py +242 -91
- data/vendor/pygments-main/pygments/lexers/math.py +2 -1
- data/vendor/pygments-main/pygments/lexers/matlab.py +10 -12
- data/vendor/pygments-main/pygments/lexers/maxima.py +2 -3
- data/vendor/pygments-main/pygments/lexers/meson.py +2 -3
- data/vendor/pygments-main/pygments/lexers/mime.py +4 -4
- data/vendor/pygments-main/pygments/lexers/minecraft.py +24 -27
- data/vendor/pygments-main/pygments/lexers/mips.py +3 -1
- data/vendor/pygments-main/pygments/lexers/ml.py +59 -61
- data/vendor/pygments-main/pygments/lexers/modeling.py +19 -22
- data/vendor/pygments-main/pygments/lexers/modula2.py +2 -3
- data/vendor/pygments-main/pygments/lexers/mojo.py +704 -0
- data/vendor/pygments-main/pygments/lexers/monte.py +2 -3
- data/vendor/pygments-main/pygments/lexers/mosel.py +3 -3
- data/vendor/pygments-main/pygments/lexers/ncl.py +4 -3
- data/vendor/pygments-main/pygments/lexers/nimrod.py +7 -8
- data/vendor/pygments-main/pygments/lexers/nit.py +2 -3
- data/vendor/pygments-main/pygments/lexers/nix.py +40 -31
- data/vendor/pygments-main/pygments/lexers/oberon.py +3 -3
- data/vendor/pygments-main/pygments/lexers/objective.py +14 -6
- data/vendor/pygments-main/pygments/lexers/ooc.py +2 -3
- data/vendor/pygments-main/pygments/lexers/openscad.py +96 -0
- data/vendor/pygments-main/pygments/lexers/other.py +2 -1
- data/vendor/pygments-main/pygments/lexers/parasail.py +2 -3
- data/vendor/pygments-main/pygments/lexers/parsers.py +41 -44
- data/vendor/pygments-main/pygments/lexers/pascal.py +6 -3
- data/vendor/pygments-main/pygments/lexers/pawn.py +5 -5
- data/vendor/pygments-main/pygments/lexers/perl.py +3 -3
- data/vendor/pygments-main/pygments/lexers/phix.py +2 -3
- data/vendor/pygments-main/pygments/lexers/php.py +4 -5
- data/vendor/pygments-main/pygments/lexers/pointless.py +2 -3
- data/vendor/pygments-main/pygments/lexers/pony.py +3 -3
- data/vendor/pygments-main/pygments/lexers/praat.py +2 -3
- data/vendor/pygments-main/pygments/lexers/procfile.py +2 -3
- data/vendor/pygments-main/pygments/lexers/prolog.py +23 -9
- data/vendor/pygments-main/pygments/lexers/promql.py +2 -1
- data/vendor/pygments-main/pygments/lexers/prql.py +251 -0
- data/vendor/pygments-main/pygments/lexers/ptx.py +119 -0
- data/vendor/pygments-main/pygments/lexers/python.py +24 -24
- data/vendor/pygments-main/pygments/lexers/q.py +5 -6
- data/vendor/pygments-main/pygments/lexers/qlik.py +3 -3
- data/vendor/pygments-main/pygments/lexers/qvt.py +5 -3
- data/vendor/pygments-main/pygments/lexers/r.py +8 -6
- data/vendor/pygments-main/pygments/lexers/rdf.py +19 -13
- data/vendor/pygments-main/pygments/lexers/rebol.py +7 -7
- data/vendor/pygments-main/pygments/lexers/resource.py +5 -6
- data/vendor/pygments-main/pygments/lexers/ride.py +4 -5
- data/vendor/pygments-main/pygments/lexers/rita.py +2 -3
- data/vendor/pygments-main/pygments/lexers/rnc.py +2 -3
- data/vendor/pygments-main/pygments/lexers/roboconf.py +5 -5
- data/vendor/pygments-main/pygments/lexers/robotframework.py +2 -3
- data/vendor/pygments-main/pygments/lexers/ruby.py +8 -13
- data/vendor/pygments-main/pygments/lexers/rust.py +9 -10
- data/vendor/pygments-main/pygments/lexers/sas.py +3 -3
- data/vendor/pygments-main/pygments/lexers/savi.py +131 -130
- data/vendor/pygments-main/pygments/lexers/scdoc.py +13 -7
- data/vendor/pygments-main/pygments/lexers/scripting.py +354 -42
- data/vendor/pygments-main/pygments/lexers/sgf.py +2 -3
- data/vendor/pygments-main/pygments/lexers/shell.py +107 -129
- data/vendor/pygments-main/pygments/lexers/sieve.py +3 -3
- data/vendor/pygments-main/pygments/lexers/slash.py +4 -5
- data/vendor/pygments-main/pygments/lexers/smalltalk.py +3 -5
- data/vendor/pygments-main/pygments/lexers/smithy.py +2 -3
- data/vendor/pygments-main/pygments/lexers/smv.py +3 -3
- data/vendor/pygments-main/pygments/lexers/snobol.py +3 -3
- data/vendor/pygments-main/pygments/lexers/solidity.py +3 -3
- data/vendor/pygments-main/pygments/lexers/soong.py +78 -0
- data/vendor/pygments-main/pygments/lexers/sophia.py +3 -4
- data/vendor/pygments-main/pygments/lexers/special.py +8 -3
- data/vendor/pygments-main/pygments/lexers/spice.py +15 -16
- data/vendor/pygments-main/pygments/lexers/sql.py +24 -18
- data/vendor/pygments-main/pygments/lexers/srcinfo.py +3 -3
- data/vendor/pygments-main/pygments/lexers/stata.py +2 -3
- data/vendor/pygments-main/pygments/lexers/supercollider.py +2 -3
- data/vendor/pygments-main/pygments/lexers/tact.py +303 -0
- data/vendor/pygments-main/pygments/lexers/tal.py +13 -10
- data/vendor/pygments-main/pygments/lexers/tcl.py +2 -3
- data/vendor/pygments-main/pygments/lexers/teal.py +2 -3
- data/vendor/pygments-main/pygments/lexers/templates.py +131 -72
- data/vendor/pygments-main/pygments/lexers/teraterm.py +2 -3
- data/vendor/pygments-main/pygments/lexers/testing.py +4 -5
- data/vendor/pygments-main/pygments/lexers/text.py +2 -1
- data/vendor/pygments-main/pygments/lexers/textedit.py +9 -6
- data/vendor/pygments-main/pygments/lexers/textfmts.py +20 -15
- data/vendor/pygments-main/pygments/lexers/theorem.py +41 -115
- data/vendor/pygments-main/pygments/lexers/thingsdb.py +50 -26
- data/vendor/pygments-main/pygments/lexers/tlb.py +3 -1
- data/vendor/pygments-main/pygments/lexers/tls.py +54 -0
- data/vendor/pygments-main/pygments/lexers/tnt.py +2 -3
- data/vendor/pygments-main/pygments/lexers/trafficscript.py +3 -3
- data/vendor/pygments-main/pygments/lexers/typoscript.py +6 -7
- data/vendor/pygments-main/pygments/lexers/typst.py +104 -0
- data/vendor/pygments-main/pygments/lexers/ul4.py +47 -5
- data/vendor/pygments-main/pygments/lexers/unicon.py +9 -7
- data/vendor/pygments-main/pygments/lexers/urbi.py +3 -3
- data/vendor/pygments-main/pygments/lexers/usd.py +6 -11
- data/vendor/pygments-main/pygments/lexers/varnish.py +5 -5
- data/vendor/pygments-main/pygments/lexers/verification.py +4 -5
- data/vendor/pygments-main/pygments/lexers/verifpal.py +65 -0
- data/vendor/pygments-main/pygments/lexers/vip.py +150 -0
- data/vendor/pygments-main/pygments/lexers/vyper.py +140 -0
- data/vendor/pygments-main/pygments/lexers/web.py +2 -1
- data/vendor/pygments-main/pygments/lexers/webassembly.py +2 -3
- data/vendor/pygments-main/pygments/lexers/webidl.py +2 -3
- data/vendor/pygments-main/pygments/lexers/webmisc.py +11 -15
- data/vendor/pygments-main/pygments/lexers/wgsl.py +6 -7
- data/vendor/pygments-main/pygments/lexers/whiley.py +2 -3
- data/vendor/pygments-main/pygments/lexers/wowtoc.py +3 -3
- data/vendor/pygments-main/pygments/lexers/wren.py +3 -4
- data/vendor/pygments-main/pygments/lexers/x10.py +6 -7
- data/vendor/pygments-main/pygments/lexers/xorg.py +2 -1
- data/vendor/pygments-main/pygments/lexers/yang.py +2 -3
- data/vendor/pygments-main/pygments/lexers/yara.py +69 -0
- data/vendor/pygments-main/pygments/lexers/zig.py +2 -1
- data/vendor/pygments-main/pygments/modeline.py +4 -4
- data/vendor/pygments-main/pygments/plugin.py +3 -19
- data/vendor/pygments-main/pygments/regexopt.py +1 -1
- data/vendor/pygments-main/pygments/scanner.py +1 -1
- data/vendor/pygments-main/pygments/sphinxext.py +38 -8
- data/vendor/pygments-main/pygments/style.py +8 -2
- data/vendor/pygments-main/pygments/styles/__init__.py +17 -59
- data/vendor/pygments-main/pygments/styles/_mapping.py +54 -0
- data/vendor/pygments-main/pygments/styles/abap.py +5 -1
- data/vendor/pygments-main/pygments/styles/algol.py +5 -1
- data/vendor/pygments-main/pygments/styles/algol_nu.py +5 -1
- data/vendor/pygments-main/pygments/styles/arduino.py +76 -72
- data/vendor/pygments-main/pygments/styles/autumn.py +6 -1
- data/vendor/pygments-main/pygments/styles/borland.py +6 -1
- data/vendor/pygments-main/pygments/styles/bw.py +6 -1
- data/vendor/pygments-main/pygments/styles/coffee.py +80 -0
- data/vendor/pygments-main/pygments/styles/colorful.py +6 -1
- data/vendor/pygments-main/pygments/styles/default.py +6 -1
- data/vendor/pygments-main/pygments/styles/dracula.py +68 -80
- data/vendor/pygments-main/pygments/styles/emacs.py +6 -1
- data/vendor/pygments-main/pygments/styles/friendly.py +6 -1
- data/vendor/pygments-main/pygments/styles/friendly_grayscale.py +6 -1
- data/vendor/pygments-main/pygments/styles/fruity.py +7 -1
- data/vendor/pygments-main/pygments/styles/gh_dark.py +7 -1
- data/vendor/pygments-main/pygments/styles/gruvbox.py +10 -1
- data/vendor/pygments-main/pygments/styles/igor.py +6 -1
- data/vendor/pygments-main/pygments/styles/inkpot.py +7 -2
- data/vendor/pygments-main/pygments/styles/lightbulb.py +110 -0
- data/vendor/pygments-main/pygments/styles/lilypond.py +8 -2
- data/vendor/pygments-main/pygments/styles/lovelace.py +16 -10
- data/vendor/pygments-main/pygments/styles/manni.py +7 -2
- data/vendor/pygments-main/pygments/styles/material.py +21 -14
- data/vendor/pygments-main/pygments/styles/monokai.py +16 -10
- data/vendor/pygments-main/pygments/styles/murphy.py +7 -2
- data/vendor/pygments-main/pygments/styles/native.py +10 -5
- data/vendor/pygments-main/pygments/styles/nord.py +9 -3
- data/vendor/pygments-main/pygments/styles/onedark.py +6 -2
- data/vendor/pygments-main/pygments/styles/paraiso_dark.py +7 -2
- data/vendor/pygments-main/pygments/styles/paraiso_light.py +7 -2
- data/vendor/pygments-main/pygments/styles/pastie.py +7 -1
- data/vendor/pygments-main/pygments/styles/perldoc.py +7 -1
- data/vendor/pygments-main/pygments/styles/rainbow_dash.py +26 -19
- data/vendor/pygments-main/pygments/styles/rrt.py +8 -2
- data/vendor/pygments-main/pygments/styles/sas.py +6 -1
- data/vendor/pygments-main/pygments/styles/solarized.py +9 -1
- data/vendor/pygments-main/pygments/styles/staroffice.py +6 -1
- data/vendor/pygments-main/pygments/styles/stata_dark.py +6 -2
- data/vendor/pygments-main/pygments/styles/stata_light.py +6 -1
- data/vendor/pygments-main/pygments/styles/tango.py +7 -3
- data/vendor/pygments-main/pygments/styles/trac.py +7 -1
- data/vendor/pygments-main/pygments/styles/vim.py +7 -1
- data/vendor/pygments-main/pygments/styles/vs.py +7 -2
- data/vendor/pygments-main/pygments/styles/xcode.py +6 -1
- data/vendor/pygments-main/pygments/styles/zenburn.py +6 -1
- data/vendor/pygments-main/pygments/token.py +2 -1
- data/vendor/pygments-main/pygments/unistring.py +5 -5
- data/vendor/pygments-main/pygments/util.py +13 -19
- data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info}/METADATA +14 -13
- data/vendor/pygments-main/pygments-2.18.0.dist-info/RECORD +663 -0
- data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info}/WHEEL +1 -2
- data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info/licenses}/AUTHORS +13 -0
- metadata +39 -15
- data/vendor/pygments-main/Pygments-2.15.1.dist-info/RECORD +0 -614
- data/vendor/pygments-main/Pygments-2.15.1.dist-info/top_level.txt +0 -1
- /data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info}/INSTALLER +0 -0
- /data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info}/REQUESTED +0 -0
- /data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info}/entry_points.txt +0 -0
- /data/vendor/pygments-main/{Pygments-2.15.1.dist-info → pygments-2.18.0.dist-info/licenses}/LICENSE +0 -0
@@ -4,12 +4,14 @@
|
|
4
4
|
|
5
5
|
A formatter that generates RTF files.
|
6
6
|
|
7
|
-
:copyright: Copyright 2006-
|
7
|
+
:copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
|
8
8
|
:license: BSD, see LICENSE for details.
|
9
9
|
"""
|
10
10
|
|
11
|
+
from collections import OrderedDict
|
11
12
|
from pygments.formatter import Formatter
|
12
|
-
from pygments.
|
13
|
+
from pygments.style import _ansimap
|
14
|
+
from pygments.util import get_bool_opt, get_int_opt, get_list_opt, surrogatepair
|
13
15
|
|
14
16
|
|
15
17
|
__all__ = ['RtfFormatter']
|
@@ -42,6 +44,59 @@ class RtfFormatter(Formatter):
|
|
42
44
|
default is 24 half-points, giving a size 12 font.
|
43
45
|
|
44
46
|
.. versionadded:: 2.0
|
47
|
+
|
48
|
+
`linenos`
|
49
|
+
Turn on line numbering (default: ``False``).
|
50
|
+
|
51
|
+
.. versionadded:: 2.18
|
52
|
+
|
53
|
+
`lineno_fontsize`
|
54
|
+
Font size for line numbers. Size is specified in half points
|
55
|
+
(default: `fontsize`).
|
56
|
+
|
57
|
+
.. versionadded:: 2.18
|
58
|
+
|
59
|
+
`lineno_padding`
|
60
|
+
Number of spaces between the (inline) line numbers and the
|
61
|
+
source code (default: ``2``).
|
62
|
+
|
63
|
+
.. versionadded:: 2.18
|
64
|
+
|
65
|
+
`linenostart`
|
66
|
+
The line number for the first line (default: ``1``).
|
67
|
+
|
68
|
+
.. versionadded:: 2.18
|
69
|
+
|
70
|
+
`linenostep`
|
71
|
+
If set to a number n > 1, only every nth line number is printed.
|
72
|
+
|
73
|
+
.. versionadded:: 2.18
|
74
|
+
|
75
|
+
`lineno_color`
|
76
|
+
Color for line numbers specified as a hex triplet, e.g. ``'5e5e5e'``.
|
77
|
+
Defaults to the style's line number color if it is a hex triplet,
|
78
|
+
otherwise ansi bright black.
|
79
|
+
|
80
|
+
.. versionadded:: 2.18
|
81
|
+
|
82
|
+
`hl_lines`
|
83
|
+
Specify a list of lines to be highlighted, as line numbers separated by
|
84
|
+
spaces, e.g. ``'3 7 8'``. The line numbers are relative to the input
|
85
|
+
(i.e. the first line is line 1) unless `hl_linenostart` is set.
|
86
|
+
|
87
|
+
.. versionadded:: 2.18
|
88
|
+
|
89
|
+
`hl_color`
|
90
|
+
Color for highlighting the lines specified in `hl_lines`, specified as
|
91
|
+
a hex triplet (default: style's `highlight_color`).
|
92
|
+
|
93
|
+
.. versionadded:: 2.18
|
94
|
+
|
95
|
+
`hl_linenostart`
|
96
|
+
If set to ``True`` line numbers in `hl_lines` are specified
|
97
|
+
relative to `linenostart` (default ``False``).
|
98
|
+
|
99
|
+
.. versionadded:: 2.18
|
45
100
|
"""
|
46
101
|
name = 'RTF'
|
47
102
|
aliases = ['rtf']
|
@@ -62,6 +117,40 @@ class RtfFormatter(Formatter):
|
|
62
117
|
Formatter.__init__(self, **options)
|
63
118
|
self.fontface = options.get('fontface') or ''
|
64
119
|
self.fontsize = get_int_opt(options, 'fontsize', 0)
|
120
|
+
self.linenos = get_bool_opt(options, 'linenos', False)
|
121
|
+
self.lineno_fontsize = get_int_opt(options, 'lineno_fontsize',
|
122
|
+
self.fontsize)
|
123
|
+
self.lineno_padding = get_int_opt(options, 'lineno_padding', 2)
|
124
|
+
self.linenostart = abs(get_int_opt(options, 'linenostart', 1))
|
125
|
+
self.linenostep = abs(get_int_opt(options, 'linenostep', 1))
|
126
|
+
self.hl_linenostart = get_bool_opt(options, 'hl_linenostart', False)
|
127
|
+
|
128
|
+
self.hl_color = options.get('hl_color', '')
|
129
|
+
if not self.hl_color:
|
130
|
+
self.hl_color = self.style.highlight_color
|
131
|
+
|
132
|
+
self.hl_lines = []
|
133
|
+
for lineno in get_list_opt(options, 'hl_lines', []):
|
134
|
+
try:
|
135
|
+
lineno = int(lineno)
|
136
|
+
if self.hl_linenostart:
|
137
|
+
lineno = lineno - self.linenostart + 1
|
138
|
+
self.hl_lines.append(lineno)
|
139
|
+
except ValueError:
|
140
|
+
pass
|
141
|
+
|
142
|
+
self.lineno_color = options.get('lineno_color', '')
|
143
|
+
if not self.lineno_color:
|
144
|
+
if self.style.line_number_color == 'inherit':
|
145
|
+
# style color is the css value 'inherit'
|
146
|
+
# default to ansi bright-black
|
147
|
+
self.lineno_color = _ansimap['ansibrightblack']
|
148
|
+
else:
|
149
|
+
# style color is assumed to be a hex triplet as other
|
150
|
+
# colors in pygments/style.py
|
151
|
+
self.lineno_color = self.style.line_number_color
|
152
|
+
|
153
|
+
self.color_mapping = self._create_color_mapping()
|
65
154
|
|
66
155
|
def _escape(self, text):
|
67
156
|
return text.replace('\\', '\\\\') \
|
@@ -90,43 +179,145 @@ class RtfFormatter(Formatter):
|
|
90
179
|
# Force surrogate pairs
|
91
180
|
buf.append('{\\u%d}{\\u%d}' % surrogatepair(cn))
|
92
181
|
|
93
|
-
return ''.join(buf).replace('\n', '\\par
|
182
|
+
return ''.join(buf).replace('\n', '\\par')
|
94
183
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
184
|
+
@staticmethod
|
185
|
+
def hex_to_rtf_color(hex_color):
|
186
|
+
if hex_color[0] == "#":
|
187
|
+
hex_color = hex_color[1:]
|
188
|
+
|
189
|
+
return '\\red%d\\green%d\\blue%d;' % (
|
190
|
+
int(hex_color[0:2], 16),
|
191
|
+
int(hex_color[2:4], 16),
|
192
|
+
int(hex_color[4:6], 16)
|
193
|
+
)
|
194
|
+
|
195
|
+
def _split_tokens_on_newlines(self, tokensource):
|
196
|
+
"""
|
197
|
+
Split tokens containing newline characters into multiple token
|
198
|
+
each representing a line of the input file. Needed for numbering
|
199
|
+
lines of e.g. multiline comments.
|
200
|
+
"""
|
201
|
+
for ttype, value in tokensource:
|
202
|
+
if value == '\n':
|
203
|
+
yield (ttype, value)
|
204
|
+
elif "\n" in value:
|
205
|
+
lines = value.split("\n")
|
206
|
+
for line in lines[:-1]:
|
207
|
+
yield (ttype, line+"\n")
|
208
|
+
if lines[-1]:
|
209
|
+
yield (ttype, lines[-1])
|
210
|
+
else:
|
211
|
+
yield (ttype, value)
|
212
|
+
|
213
|
+
def _create_color_mapping(self):
|
214
|
+
"""
|
215
|
+
Create a mapping of style hex colors to index/offset in
|
216
|
+
the RTF color table.
|
217
|
+
"""
|
218
|
+
color_mapping = OrderedDict()
|
105
219
|
offset = 1
|
220
|
+
|
221
|
+
if self.linenos:
|
222
|
+
color_mapping[self.lineno_color] = offset
|
223
|
+
offset += 1
|
224
|
+
|
225
|
+
if self.hl_lines:
|
226
|
+
color_mapping[self.hl_color] = offset
|
227
|
+
offset += 1
|
228
|
+
|
106
229
|
for _, style in self.style:
|
107
230
|
for color in style['color'], style['bgcolor'], style['border']:
|
108
231
|
if color and color not in color_mapping:
|
109
232
|
color_mapping[color] = offset
|
110
|
-
outfile.write('\\red%d\\green%d\\blue%d;' % (
|
111
|
-
int(color[0:2], 16),
|
112
|
-
int(color[2:4], 16),
|
113
|
-
int(color[4:6], 16)
|
114
|
-
))
|
115
233
|
offset += 1
|
116
|
-
|
234
|
+
|
235
|
+
return color_mapping
|
236
|
+
|
237
|
+
@property
|
238
|
+
def _lineno_template(self):
|
239
|
+
if self.lineno_fontsize != self.fontsize:
|
240
|
+
return '{{\\fs{} \\cf{} %s{}}}'.format(self.lineno_fontsize,
|
241
|
+
self.color_mapping[self.lineno_color],
|
242
|
+
" " * self.lineno_padding)
|
243
|
+
|
244
|
+
return '{{\\cf{} %s{}}}'.format(self.color_mapping[self.lineno_color],
|
245
|
+
" " * self.lineno_padding)
|
246
|
+
|
247
|
+
@property
|
248
|
+
def _hl_open_str(self):
|
249
|
+
return rf'{{\highlight{self.color_mapping[self.hl_color]} '
|
250
|
+
|
251
|
+
@property
|
252
|
+
def _rtf_header(self):
|
253
|
+
lines = []
|
254
|
+
# rtf 1.8 header
|
255
|
+
lines.append('{\\rtf1\\ansi\\uc0\\deff0'
|
256
|
+
'{\\fonttbl{\\f0\\fmodern\\fprq1\\fcharset0%s;}}'
|
257
|
+
% (self.fontface and ' '
|
258
|
+
+ self._escape(self.fontface) or ''))
|
259
|
+
|
260
|
+
# color table
|
261
|
+
lines.append('{\\colortbl;')
|
262
|
+
for color, _ in self.color_mapping.items():
|
263
|
+
lines.append(self.hex_to_rtf_color(color))
|
264
|
+
lines.append('}')
|
265
|
+
|
266
|
+
# font and fontsize
|
267
|
+
lines.append('\\f0\\sa0')
|
117
268
|
if self.fontsize:
|
118
|
-
|
269
|
+
lines.append('\\fs%d' % self.fontsize)
|
270
|
+
|
271
|
+
# ensure Libre Office Writer imports and renders consecutive
|
272
|
+
# space characters the same width, needed for line numbering.
|
273
|
+
# https://bugs.documentfoundation.org/show_bug.cgi?id=144050
|
274
|
+
lines.append('\\dntblnsbdb')
|
275
|
+
|
276
|
+
return lines
|
277
|
+
|
278
|
+
def format_unencoded(self, tokensource, outfile):
|
279
|
+
for line in self._rtf_header:
|
280
|
+
outfile.write(line + "\n")
|
281
|
+
|
282
|
+
tokensource = self._split_tokens_on_newlines(tokensource)
|
283
|
+
|
284
|
+
# first pass of tokens to count lines, needed for line numbering
|
285
|
+
if self.linenos:
|
286
|
+
line_count = 0
|
287
|
+
tokens = [] # for copying the token source generator
|
288
|
+
for ttype, value in tokensource:
|
289
|
+
tokens.append((ttype, value))
|
290
|
+
if value.endswith("\n"):
|
291
|
+
line_count += 1
|
292
|
+
|
293
|
+
# width of line number strings (for padding with spaces)
|
294
|
+
linenos_width = len(str(line_count+self.linenostart-1))
|
295
|
+
|
296
|
+
tokensource = tokens
|
119
297
|
|
120
298
|
# highlight stream
|
299
|
+
lineno = 1
|
300
|
+
start_new_line = True
|
121
301
|
for ttype, value in tokensource:
|
302
|
+
if start_new_line and lineno in self.hl_lines:
|
303
|
+
outfile.write(self._hl_open_str)
|
304
|
+
|
305
|
+
if start_new_line and self.linenos:
|
306
|
+
if (lineno-self.linenostart+1)%self.linenostep == 0:
|
307
|
+
current_lineno = lineno + self.linenostart - 1
|
308
|
+
lineno_str = str(current_lineno).rjust(linenos_width)
|
309
|
+
else:
|
310
|
+
lineno_str = "".rjust(linenos_width)
|
311
|
+
outfile.write(self._lineno_template % lineno_str)
|
312
|
+
|
122
313
|
while not self.style.styles_token(ttype) and ttype.parent:
|
123
314
|
ttype = ttype.parent
|
124
315
|
style = self.style.style_for_token(ttype)
|
125
316
|
buf = []
|
126
317
|
if style['bgcolor']:
|
127
|
-
buf.append('\\cb%d' % color_mapping[style['bgcolor']])
|
318
|
+
buf.append('\\cb%d' % self.color_mapping[style['bgcolor']])
|
128
319
|
if style['color']:
|
129
|
-
buf.append('\\cf%d' % color_mapping[style['color']])
|
320
|
+
buf.append('\\cf%d' % self.color_mapping[style['color']])
|
130
321
|
if style['bold']:
|
131
322
|
buf.append('\\b')
|
132
323
|
if style['italic']:
|
@@ -135,12 +326,24 @@ class RtfFormatter(Formatter):
|
|
135
326
|
buf.append('\\ul')
|
136
327
|
if style['border']:
|
137
328
|
buf.append('\\chbrdr\\chcfpat%d' %
|
138
|
-
color_mapping[style['border']])
|
329
|
+
self.color_mapping[style['border']])
|
139
330
|
start = ''.join(buf)
|
140
331
|
if start:
|
141
|
-
outfile.write('{
|
332
|
+
outfile.write(f'{{{start} ')
|
142
333
|
outfile.write(self._escape_text(value))
|
143
334
|
if start:
|
144
335
|
outfile.write('}')
|
336
|
+
start_new_line = False
|
337
|
+
|
338
|
+
# complete line of input
|
339
|
+
if value.endswith("\n"):
|
340
|
+
# close line highlighting
|
341
|
+
if lineno in self.hl_lines:
|
342
|
+
outfile.write('}')
|
343
|
+
# newline in RTF file after closing }
|
344
|
+
outfile.write("\n")
|
345
|
+
|
346
|
+
start_new_line = True
|
347
|
+
lineno += 1
|
145
348
|
|
146
|
-
outfile.write('}')
|
349
|
+
outfile.write('}\n')
|
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
Formatter for SVG output.
|
6
6
|
|
7
|
-
:copyright: Copyright 2006-
|
7
|
+
:copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
|
8
8
|
:license: BSD, see LICENSE for details.
|
9
9
|
"""
|
10
10
|
|
@@ -60,11 +60,11 @@ class SvgFormatter(Formatter):
|
|
60
60
|
|
61
61
|
`linenostep`
|
62
62
|
If set to a number n > 1, only every nth line number is printed.
|
63
|
-
|
63
|
+
|
64
64
|
`linenowidth`
|
65
65
|
Maximum width devoted to line numbers (default: ``3*ystep``, sufficient
|
66
|
-
for up to 4-digit line numbers. Increase width for longer code blocks).
|
67
|
-
|
66
|
+
for up to 4-digit line numbers. Increase width for longer code blocks).
|
67
|
+
|
68
68
|
`xoffset`
|
69
69
|
Starting offset in X direction, defaults to ``0``.
|
70
70
|
|
@@ -97,10 +97,11 @@ class SvgFormatter(Formatter):
|
|
97
97
|
self.fontsize = options.get('fontsize', '14px')
|
98
98
|
self.xoffset = get_int_opt(options, 'xoffset', 0)
|
99
99
|
fs = self.fontsize.strip()
|
100
|
-
if fs.endswith('px'):
|
100
|
+
if fs.endswith('px'):
|
101
|
+
fs = fs[:-2].strip()
|
101
102
|
try:
|
102
103
|
int_fs = int(fs)
|
103
|
-
except:
|
104
|
+
except ValueError:
|
104
105
|
int_fs = 20
|
105
106
|
self.yoffset = get_int_opt(options, 'yoffset', int_fs)
|
106
107
|
self.ystep = get_int_opt(options, 'ystep', int_fs + 5)
|
@@ -122,30 +123,27 @@ class SvgFormatter(Formatter):
|
|
122
123
|
y = self.yoffset
|
123
124
|
if not self.nowrap:
|
124
125
|
if self.encoding:
|
125
|
-
outfile.write('<?xml version="1.0" encoding="
|
126
|
-
self.encoding)
|
126
|
+
outfile.write(f'<?xml version="1.0" encoding="{self.encoding}"?>\n')
|
127
127
|
else:
|
128
128
|
outfile.write('<?xml version="1.0"?>\n')
|
129
129
|
outfile.write('<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" '
|
130
130
|
'"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/'
|
131
131
|
'svg10.dtd">\n')
|
132
132
|
outfile.write('<svg xmlns="http://www.w3.org/2000/svg">\n')
|
133
|
-
outfile.write('<g font-family="
|
134
|
-
|
135
|
-
|
136
|
-
counter = self.linenostart
|
133
|
+
outfile.write(f'<g font-family="{self.fontfamily}" font-size="{self.fontsize}">\n')
|
134
|
+
|
135
|
+
counter = self.linenostart
|
137
136
|
counter_step = self.linenostep
|
138
137
|
counter_style = self._get_style(Comment)
|
139
138
|
line_x = x
|
140
|
-
|
139
|
+
|
141
140
|
if self.linenos:
|
142
141
|
if counter % counter_step == 0:
|
143
|
-
outfile.write('<text x="
|
144
|
-
(x+self.linenowidth,y,counter_style,counter))
|
142
|
+
outfile.write(f'<text x="{x+self.linenowidth}" y="{y}" {counter_style} text-anchor="end">{counter}</text>')
|
145
143
|
line_x += self.linenowidth + self.ystep
|
146
144
|
counter += 1
|
147
145
|
|
148
|
-
outfile.write('<text x="
|
146
|
+
outfile.write(f'<text x="{line_x}" y="{y}" xml:space="preserve">')
|
149
147
|
for ttype, value in tokensource:
|
150
148
|
style = self._get_style(ttype)
|
151
149
|
tspan = style and '<tspan' + style + '>' or ''
|
@@ -159,11 +157,10 @@ class SvgFormatter(Formatter):
|
|
159
157
|
y += self.ystep
|
160
158
|
outfile.write('</text>\n')
|
161
159
|
if self.linenos and counter % counter_step == 0:
|
162
|
-
outfile.write('<text x="
|
163
|
-
|
164
|
-
|
160
|
+
outfile.write(f'<text x="{x+self.linenowidth}" y="{y}" text-anchor="end" {counter_style}>{counter}</text>')
|
161
|
+
|
165
162
|
counter += 1
|
166
|
-
outfile.write('<text x="
|
163
|
+
outfile.write(f'<text x="{line_x}" y="{y}" ' 'xml:space="preserve">')
|
167
164
|
outfile.write(tspan + parts[-1] + tspanend)
|
168
165
|
outfile.write('</text>')
|
169
166
|
|
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
Base lexer classes.
|
6
6
|
|
7
|
-
:copyright: Copyright 2006-
|
7
|
+
:copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
|
8
8
|
:license: BSD, see LICENSE for details.
|
9
9
|
"""
|
10
10
|
|
@@ -67,10 +67,17 @@ class Lexer(metaclass=LexerMeta):
|
|
67
67
|
:no-value:
|
68
68
|
.. autoattribute:: priority
|
69
69
|
|
70
|
-
Lexers included in Pygments should have
|
70
|
+
Lexers included in Pygments should have two additional attributes:
|
71
71
|
|
72
72
|
.. autoattribute:: url
|
73
73
|
:no-value:
|
74
|
+
.. autoattribute:: version_added
|
75
|
+
:no-value:
|
76
|
+
|
77
|
+
Lexers included in Pygments may have additional attributes:
|
78
|
+
|
79
|
+
.. autoattribute:: _example
|
80
|
+
:no-value:
|
74
81
|
|
75
82
|
You can pass options to the constructor. The basic options recognized
|
76
83
|
by all lexers and processed by the base `Lexer` class are:
|
@@ -125,9 +132,16 @@ class Lexer(metaclass=LexerMeta):
|
|
125
132
|
priority = 0
|
126
133
|
|
127
134
|
#: URL of the language specification/definition. Used in the Pygments
|
128
|
-
#: documentation.
|
135
|
+
#: documentation. Set to an empty string to disable.
|
129
136
|
url = None
|
130
137
|
|
138
|
+
#: Version of Pygments in which the lexer was added.
|
139
|
+
version_added = None
|
140
|
+
|
141
|
+
#: Example file name. Relative to the ``tests/examplefiles`` directory.
|
142
|
+
#: This is used by the documentation generator to show an example.
|
143
|
+
_example = None
|
144
|
+
|
131
145
|
def __init__(self, **options):
|
132
146
|
"""
|
133
147
|
This constructor takes arbitrary options as keyword arguments.
|
@@ -160,10 +174,9 @@ class Lexer(metaclass=LexerMeta):
|
|
160
174
|
|
161
175
|
def __repr__(self):
|
162
176
|
if self.options:
|
163
|
-
return '<pygments.lexers
|
164
|
-
self.options)
|
177
|
+
return f'<pygments.lexers.{self.__class__.__name__} with {self.options!r}>'
|
165
178
|
else:
|
166
|
-
return '<pygments.lexers
|
179
|
+
return f'<pygments.lexers.{self.__class__.__name__}>'
|
167
180
|
|
168
181
|
def add_filter(self, filter_, **options):
|
169
182
|
"""
|
@@ -190,20 +203,9 @@ class Lexer(metaclass=LexerMeta):
|
|
190
203
|
it's the same as if the return values was ``0.0``.
|
191
204
|
"""
|
192
205
|
|
193
|
-
def
|
194
|
-
"""
|
195
|
-
This method is the basic interface of a lexer. It is called by
|
196
|
-
the `highlight()` function. It must process the text and return an
|
197
|
-
iterable of ``(tokentype, value)`` pairs from `text`.
|
198
|
-
|
199
|
-
Normally, you don't need to override this method. The default
|
200
|
-
implementation processes the options recognized by all lexers
|
201
|
-
(`stripnl`, `stripall` and so on), and then yields all tokens
|
202
|
-
from `get_tokens_unprocessed()`, with the ``index`` dropped.
|
206
|
+
def _preprocess_lexer_input(self, text):
|
207
|
+
"""Apply preprocessing such as decoding the input, removing BOM and normalizing newlines."""
|
203
208
|
|
204
|
-
If `unfiltered` is set to `True`, the filtering mechanism is
|
205
|
-
bypassed even if filters are defined.
|
206
|
-
"""
|
207
209
|
if not isinstance(text, str):
|
208
210
|
if self.encoding == 'guess':
|
209
211
|
text, _ = guess_decode(text)
|
@@ -246,6 +248,24 @@ class Lexer(metaclass=LexerMeta):
|
|
246
248
|
if self.ensurenl and not text.endswith('\n'):
|
247
249
|
text += '\n'
|
248
250
|
|
251
|
+
return text
|
252
|
+
|
253
|
+
def get_tokens(self, text, unfiltered=False):
|
254
|
+
"""
|
255
|
+
This method is the basic interface of a lexer. It is called by
|
256
|
+
the `highlight()` function. It must process the text and return an
|
257
|
+
iterable of ``(tokentype, value)`` pairs from `text`.
|
258
|
+
|
259
|
+
Normally, you don't need to override this method. The default
|
260
|
+
implementation processes the options recognized by all lexers
|
261
|
+
(`stripnl`, `stripall` and so on), and then yields all tokens
|
262
|
+
from `get_tokens_unprocessed()`, with the ``index`` dropped.
|
263
|
+
|
264
|
+
If `unfiltered` is set to `True`, the filtering mechanism is
|
265
|
+
bypassed even if filters are defined.
|
266
|
+
"""
|
267
|
+
text = self._preprocess_lexer_input(text)
|
268
|
+
|
249
269
|
def streamer():
|
250
270
|
for _, t, v in self.get_tokens_unprocessed(text):
|
251
271
|
yield t, v
|
@@ -490,7 +510,7 @@ class RegexLexerMeta(LexerMeta):
|
|
490
510
|
def _process_token(cls, token):
|
491
511
|
"""Preprocess the token component of a token definition."""
|
492
512
|
assert type(token) is _TokenType or callable(token), \
|
493
|
-
'token type must be simple type or callable, not
|
513
|
+
f'token type must be simple type or callable, not {token!r}'
|
494
514
|
return token
|
495
515
|
|
496
516
|
def _process_new_state(cls, new_state, unprocessed, processed):
|
@@ -506,14 +526,14 @@ class RegexLexerMeta(LexerMeta):
|
|
506
526
|
elif new_state[:5] == '#pop:':
|
507
527
|
return -int(new_state[5:])
|
508
528
|
else:
|
509
|
-
assert False, 'unknown new state
|
529
|
+
assert False, f'unknown new state {new_state!r}'
|
510
530
|
elif isinstance(new_state, combined):
|
511
531
|
# combine a new state from existing ones
|
512
532
|
tmp_state = '_tmp_%d' % cls._tmpname
|
513
533
|
cls._tmpname += 1
|
514
534
|
itokens = []
|
515
535
|
for istate in new_state:
|
516
|
-
assert istate != new_state, 'circular state ref
|
536
|
+
assert istate != new_state, f'circular state ref {istate!r}'
|
517
537
|
itokens.extend(cls._process_state(unprocessed,
|
518
538
|
processed, istate))
|
519
539
|
processed[tmp_state] = itokens
|
@@ -526,12 +546,12 @@ class RegexLexerMeta(LexerMeta):
|
|
526
546
|
'unknown new state ' + istate
|
527
547
|
return new_state
|
528
548
|
else:
|
529
|
-
assert False, 'unknown new state def
|
549
|
+
assert False, f'unknown new state def {new_state!r}'
|
530
550
|
|
531
551
|
def _process_state(cls, unprocessed, processed, state):
|
532
552
|
"""Preprocess a single state definition."""
|
533
|
-
assert
|
534
|
-
assert state[0] != '#', "invalid state name
|
553
|
+
assert isinstance(state, str), f"wrong state name {state!r}"
|
554
|
+
assert state[0] != '#', f"invalid state name {state!r}"
|
535
555
|
if state in processed:
|
536
556
|
return processed[state]
|
537
557
|
tokens = processed[state] = []
|
@@ -539,7 +559,7 @@ class RegexLexerMeta(LexerMeta):
|
|
539
559
|
for tdef in unprocessed[state]:
|
540
560
|
if isinstance(tdef, include):
|
541
561
|
# it's a state reference
|
542
|
-
assert tdef != state, "circular state reference
|
562
|
+
assert tdef != state, f"circular state reference {state!r}"
|
543
563
|
tokens.extend(cls._process_state(unprocessed, processed,
|
544
564
|
str(tdef)))
|
545
565
|
continue
|
@@ -553,13 +573,12 @@ class RegexLexerMeta(LexerMeta):
|
|
553
573
|
tokens.append((re.compile('').match, None, new_state))
|
554
574
|
continue
|
555
575
|
|
556
|
-
assert type(tdef) is tuple, "wrong rule def
|
576
|
+
assert type(tdef) is tuple, f"wrong rule def {tdef!r}"
|
557
577
|
|
558
578
|
try:
|
559
579
|
rex = cls._process_regex(tdef[0], rflags, state)
|
560
580
|
except Exception as err:
|
561
|
-
raise ValueError("uncompilable regex
|
562
|
-
(tdef[0], state, cls, err)) from err
|
581
|
+
raise ValueError(f"uncompilable regex {tdef[0]!r} in state {state!r} of {cls!r}: {err}") from err
|
563
582
|
|
564
583
|
token = cls._process_token(tdef[1])
|
565
584
|
|
@@ -720,7 +739,7 @@ class RegexLexer(Lexer, metaclass=RegexLexerMeta):
|
|
720
739
|
elif new_state == '#push':
|
721
740
|
statestack.append(statestack[-1])
|
722
741
|
else:
|
723
|
-
assert False, "wrong state def:
|
742
|
+
assert False, f"wrong state def: {new_state!r}"
|
724
743
|
statetokens = tokendefs[statestack[-1]]
|
725
744
|
break
|
726
745
|
else:
|
@@ -752,8 +771,7 @@ class LexerContext:
|
|
752
771
|
self.stack = stack or ['root']
|
753
772
|
|
754
773
|
def __repr__(self):
|
755
|
-
return 'LexerContext(
|
756
|
-
self.text, self.pos, self.stack)
|
774
|
+
return f'LexerContext({self.text!r}, {self.pos!r}, {self.stack!r})'
|
757
775
|
|
758
776
|
|
759
777
|
class ExtendedRegexLexer(RegexLexer):
|
@@ -808,7 +826,7 @@ class ExtendedRegexLexer(RegexLexer):
|
|
808
826
|
elif new_state == '#push':
|
809
827
|
ctx.stack.append(ctx.stack[-1])
|
810
828
|
else:
|
811
|
-
assert False, "wrong state def:
|
829
|
+
assert False, f"wrong state def: {new_state!r}"
|
812
830
|
statetokens = tokendefs[ctx.stack[-1]]
|
813
831
|
break
|
814
832
|
else:
|