pygments.rb 1.2.1 → 2.0.0.rc2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/dependabot.yml +13 -0
- data/.github/workflows/ci.yml +28 -0
- data/.github/workflows/release.yml +24 -0
- data/.gitignore +5 -6
- data/CHANGELOG.adoc +119 -0
- data/Gemfile +3 -1
- data/LICENSE +1 -1
- data/README.adoc +161 -0
- data/Rakefile +10 -21
- data/bench.rb +8 -7
- data/cache-lexers.rb +3 -2
- data/lib/pygments.rb +10 -11
- data/lib/pygments/lexer.rb +5 -5
- data/lib/pygments/mentos.py +23 -66
- data/lib/pygments/popen.rb +152 -204
- data/lib/pygments/version.rb +2 -2
- data/pygments.rb.gemspec +11 -9
- data/test/test_pygments.rb +51 -84
- data/vendor/pygments-main/{AUTHORS → Pygments-2.7.3.dist-info/AUTHORS} +21 -3
- data/vendor/pygments-main/Pygments-2.7.3.dist-info/INSTALLER +1 -0
- data/vendor/pygments-main/{LICENSE → Pygments-2.7.3.dist-info/LICENSE} +1 -1
- data/vendor/pygments-main/Pygments-2.7.3.dist-info/METADATA +49 -0
- data/vendor/pygments-main/Pygments-2.7.3.dist-info/RECORD +482 -0
- data/vendor/pygments-main/Pygments-2.7.3.dist-info/REQUESTED +0 -0
- data/vendor/pygments-main/Pygments-2.7.3.dist-info/WHEEL +5 -0
- data/vendor/pygments-main/Pygments-2.7.3.dist-info/entry_points.txt +3 -0
- data/vendor/pygments-main/Pygments-2.7.3.dist-info/top_level.txt +1 -0
- data/vendor/pygments-main/bin/pygmentize +8 -0
- data/vendor/pygments-main/pygments/__init__.py +6 -11
- data/vendor/pygments-main/pygments/__main__.py +18 -0
- data/vendor/pygments-main/pygments/cmdline.py +38 -29
- data/vendor/pygments-main/pygments/console.py +6 -9
- data/vendor/pygments-main/pygments/filter.py +4 -6
- data/vendor/pygments-main/pygments/filters/__init__.py +609 -21
- data/vendor/pygments-main/pygments/formatter.py +4 -4
- data/vendor/pygments-main/pygments/formatters/__init__.py +9 -8
- data/vendor/pygments-main/pygments/formatters/_mapping.py +1 -3
- data/vendor/pygments-main/pygments/formatters/bbcode.py +1 -1
- data/vendor/pygments-main/pygments/formatters/html.py +223 -135
- data/vendor/pygments-main/pygments/formatters/img.py +68 -41
- data/vendor/pygments-main/pygments/formatters/irc.py +39 -39
- data/vendor/pygments-main/pygments/formatters/latex.py +56 -26
- data/vendor/pygments-main/pygments/formatters/other.py +12 -8
- data/vendor/pygments-main/pygments/formatters/rtf.py +29 -29
- data/vendor/pygments-main/pygments/formatters/svg.py +38 -4
- data/vendor/pygments-main/pygments/formatters/terminal.py +25 -31
- data/vendor/pygments-main/pygments/formatters/terminal256.py +22 -12
- data/vendor/pygments-main/pygments/lexer.py +41 -39
- data/vendor/pygments-main/pygments/lexers/__init__.py +342 -0
- data/vendor/pygments-main/pygments/lexers/_asy_builtins.py +1645 -0
- data/vendor/pygments-main/pygments/lexers/_cl_builtins.py +232 -0
- data/vendor/pygments-main/pygments/lexers/_cocoa_builtins.py +71 -0
- data/vendor/pygments-main/pygments/lexers/_csound_builtins.py +1725 -0
- data/vendor/pygments-main/pygments/lexers/_lasso_builtins.py +5327 -0
- data/vendor/pygments-main/pygments/lexers/_lua_builtins.py +293 -0
- data/vendor/pygments-main/pygments/lexers/_mapping.py +551 -0
- data/vendor/pygments-main/pygments/lexers/_mql_builtins.py +1172 -0
- data/vendor/pygments-main/pygments/lexers/_mysql_builtins.py +1282 -0
- data/vendor/pygments-main/pygments/lexers/_openedge_builtins.py +2547 -0
- data/vendor/pygments-main/pygments/lexers/_php_builtins.py +4753 -0
- data/vendor/pygments-main/pygments/lexers/_postgres_builtins.py +678 -0
- data/vendor/pygments-main/pygments/lexers/_scilab_builtins.py +3094 -0
- data/vendor/pygments-main/pygments/lexers/_sourcemod_builtins.py +1161 -0
- data/vendor/pygments-main/pygments/lexers/_stan_builtins.py +558 -0
- data/vendor/pygments-main/pygments/lexers/_stata_builtins.py +421 -0
- data/vendor/pygments-main/pygments/lexers/_tsql_builtins.py +1004 -0
- data/vendor/pygments-main/pygments/lexers/_usd_builtins.py +113 -0
- data/vendor/pygments-main/pygments/lexers/_vbscript_builtins.py +280 -0
- data/vendor/pygments-main/pygments/lexers/_vim_builtins.py +1939 -0
- data/vendor/pygments-main/pygments/lexers/actionscript.py +245 -0
- data/vendor/pygments-main/pygments/lexers/agile.py +24 -0
- data/vendor/pygments-main/pygments/lexers/algebra.py +240 -0
- data/vendor/pygments-main/pygments/lexers/ambient.py +76 -0
- data/vendor/pygments-main/pygments/lexers/ampl.py +87 -0
- data/vendor/pygments-main/pygments/lexers/apl.py +101 -0
- data/vendor/pygments-main/pygments/lexers/archetype.py +318 -0
- data/vendor/pygments-main/pygments/lexers/arrow.py +117 -0
- data/vendor/pygments-main/pygments/lexers/asm.py +1005 -0
- data/vendor/pygments-main/pygments/lexers/automation.py +374 -0
- data/vendor/pygments-main/pygments/lexers/bare.py +104 -0
- data/vendor/pygments-main/pygments/lexers/basic.py +662 -0
- data/vendor/pygments-main/pygments/lexers/bibtex.py +160 -0
- data/vendor/pygments-main/pygments/lexers/boa.py +102 -0
- data/vendor/pygments-main/pygments/lexers/business.py +627 -0
- data/vendor/pygments-main/pygments/lexers/c_cpp.py +344 -0
- data/vendor/pygments-main/pygments/lexers/c_like.py +566 -0
- data/vendor/pygments-main/pygments/lexers/capnproto.py +78 -0
- data/vendor/pygments-main/pygments/lexers/chapel.py +112 -0
- data/vendor/pygments-main/pygments/lexers/clean.py +179 -0
- data/vendor/pygments-main/pygments/lexers/compiled.py +34 -0
- data/vendor/pygments-main/pygments/lexers/configs.py +984 -0
- data/vendor/pygments-main/pygments/lexers/console.py +114 -0
- data/vendor/pygments-main/pygments/lexers/crystal.py +392 -0
- data/vendor/pygments-main/pygments/lexers/csound.py +467 -0
- data/vendor/pygments-main/pygments/lexers/css.py +691 -0
- data/vendor/pygments-main/pygments/lexers/d.py +256 -0
- data/vendor/pygments-main/pygments/lexers/dalvik.py +125 -0
- data/vendor/pygments-main/pygments/lexers/data.py +698 -0
- data/vendor/pygments-main/pygments/lexers/devicetree.py +109 -0
- data/vendor/pygments-main/pygments/lexers/diff.py +165 -0
- data/vendor/pygments-main/pygments/lexers/dotnet.py +707 -0
- data/vendor/pygments-main/pygments/lexers/dsls.py +960 -0
- data/vendor/pygments-main/pygments/lexers/dylan.py +287 -0
- data/vendor/pygments-main/pygments/lexers/ecl.py +139 -0
- data/vendor/pygments-main/pygments/lexers/eiffel.py +65 -0
- data/vendor/pygments-main/pygments/lexers/elm.py +121 -0
- data/vendor/pygments-main/pygments/lexers/email.py +151 -0
- data/vendor/pygments-main/pygments/lexers/erlang.py +530 -0
- data/vendor/pygments-main/pygments/lexers/esoteric.py +304 -0
- data/vendor/pygments-main/pygments/lexers/ezhil.py +77 -0
- data/vendor/pygments-main/pygments/lexers/factor.py +344 -0
- data/vendor/pygments-main/pygments/lexers/fantom.py +250 -0
- data/vendor/pygments-main/pygments/lexers/felix.py +273 -0
- data/vendor/pygments-main/pygments/lexers/floscript.py +83 -0
- data/vendor/pygments-main/pygments/lexers/forth.py +178 -0
- data/vendor/pygments-main/pygments/lexers/fortran.py +206 -0
- data/vendor/pygments-main/pygments/lexers/foxpro.py +428 -0
- data/vendor/pygments-main/pygments/lexers/freefem.py +898 -0
- data/vendor/pygments-main/pygments/lexers/functional.py +21 -0
- data/vendor/pygments-main/pygments/lexers/gdscript.py +346 -0
- data/vendor/pygments-main/pygments/lexers/go.py +101 -0
- data/vendor/pygments-main/pygments/lexers/grammar_notation.py +270 -0
- data/vendor/pygments-main/pygments/lexers/graph.py +85 -0
- data/vendor/pygments-main/pygments/lexers/graphics.py +800 -0
- data/vendor/pygments-main/pygments/lexers/haskell.py +870 -0
- data/vendor/pygments-main/pygments/lexers/haxe.py +936 -0
- data/vendor/pygments-main/pygments/lexers/hdl.py +472 -0
- data/vendor/pygments-main/pygments/lexers/hexdump.py +103 -0
- data/vendor/pygments-main/pygments/lexers/html.py +614 -0
- data/vendor/pygments-main/pygments/lexers/idl.py +281 -0
- data/vendor/pygments-main/pygments/lexers/igor.py +420 -0
- data/vendor/pygments-main/pygments/lexers/inferno.py +96 -0
- data/vendor/pygments-main/pygments/lexers/installers.py +322 -0
- data/vendor/pygments-main/pygments/lexers/int_fiction.py +1368 -0
- data/vendor/pygments-main/pygments/lexers/iolang.py +63 -0
- data/vendor/pygments-main/pygments/lexers/j.py +146 -0
- data/vendor/pygments-main/pygments/lexers/javascript.py +1540 -0
- data/vendor/pygments-main/pygments/lexers/julia.py +331 -0
- data/vendor/pygments-main/pygments/lexers/jvm.py +1673 -0
- data/vendor/pygments-main/pygments/lexers/lisp.py +2699 -0
- data/vendor/pygments-main/pygments/lexers/make.py +206 -0
- data/vendor/pygments-main/pygments/lexers/markup.py +765 -0
- data/vendor/pygments-main/pygments/lexers/math.py +21 -0
- data/vendor/pygments-main/pygments/lexers/matlab.py +720 -0
- data/vendor/pygments-main/pygments/lexers/mime.py +226 -0
- data/vendor/pygments-main/pygments/lexers/ml.py +958 -0
- data/vendor/pygments-main/pygments/lexers/modeling.py +366 -0
- data/vendor/pygments-main/pygments/lexers/modula2.py +1580 -0
- data/vendor/pygments-main/pygments/lexers/monte.py +204 -0
- data/vendor/pygments-main/pygments/lexers/mosel.py +448 -0
- data/vendor/pygments-main/pygments/lexers/ncl.py +894 -0
- data/vendor/pygments-main/pygments/lexers/nimrod.py +159 -0
- data/vendor/pygments-main/pygments/lexers/nit.py +64 -0
- data/vendor/pygments-main/pygments/lexers/nix.py +136 -0
- data/vendor/pygments-main/pygments/lexers/oberon.py +121 -0
- data/vendor/pygments-main/pygments/lexers/objective.py +504 -0
- data/vendor/pygments-main/pygments/lexers/ooc.py +85 -0
- data/vendor/pygments-main/pygments/lexers/other.py +41 -0
- data/vendor/pygments-main/pygments/lexers/parasail.py +79 -0
- data/vendor/pygments-main/pygments/lexers/parsers.py +800 -0
- data/vendor/pygments-main/pygments/lexers/pascal.py +644 -0
- data/vendor/pygments-main/pygments/lexers/pawn.py +205 -0
- data/vendor/pygments-main/pygments/lexers/perl.py +732 -0
- data/vendor/pygments-main/pygments/lexers/php.py +321 -0
- data/vendor/pygments-main/pygments/lexers/pointless.py +71 -0
- data/vendor/pygments-main/pygments/lexers/pony.py +94 -0
- data/vendor/pygments-main/pygments/lexers/praat.py +302 -0
- data/vendor/pygments-main/pygments/lexers/prolog.py +306 -0
- data/vendor/pygments-main/pygments/lexers/promql.py +183 -0
- data/vendor/pygments-main/pygments/lexers/python.py +1151 -0
- data/vendor/pygments-main/pygments/lexers/qvt.py +152 -0
- data/vendor/pygments-main/pygments/lexers/r.py +191 -0
- data/vendor/pygments-main/pygments/lexers/rdf.py +463 -0
- data/vendor/pygments-main/pygments/lexers/rebol.py +431 -0
- data/vendor/pygments-main/pygments/lexers/resource.py +85 -0
- data/vendor/pygments-main/pygments/lexers/ride.py +139 -0
- data/vendor/pygments-main/pygments/lexers/rnc.py +67 -0
- data/vendor/pygments-main/pygments/lexers/roboconf.py +82 -0
- data/vendor/pygments-main/pygments/lexers/robotframework.py +552 -0
- data/vendor/pygments-main/pygments/lexers/ruby.py +517 -0
- data/vendor/pygments-main/pygments/lexers/rust.py +224 -0
- data/vendor/pygments-main/pygments/lexers/sas.py +228 -0
- data/vendor/pygments-main/pygments/lexers/scdoc.py +83 -0
- data/vendor/pygments-main/pygments/lexers/scripting.py +1284 -0
- data/vendor/pygments-main/pygments/lexers/sgf.py +61 -0
- data/vendor/pygments-main/pygments/lexers/shell.py +914 -0
- data/vendor/pygments-main/pygments/lexers/sieve.py +69 -0
- data/vendor/pygments-main/pygments/lexers/slash.py +185 -0
- data/vendor/pygments-main/pygments/lexers/smalltalk.py +195 -0
- data/vendor/pygments-main/pygments/lexers/smv.py +79 -0
- data/vendor/pygments-main/pygments/lexers/snobol.py +83 -0
- data/vendor/pygments-main/pygments/lexers/solidity.py +92 -0
- data/vendor/pygments-main/pygments/lexers/special.py +105 -0
- data/vendor/pygments-main/pygments/lexers/sql.py +837 -0
- data/vendor/pygments-main/pygments/lexers/stata.py +171 -0
- data/vendor/pygments-main/pygments/lexers/supercollider.py +95 -0
- data/vendor/pygments-main/pygments/lexers/tcl.py +145 -0
- data/vendor/pygments-main/pygments/lexers/templates.py +2264 -0
- data/vendor/pygments-main/pygments/lexers/teraterm.py +335 -0
- data/vendor/pygments-main/pygments/lexers/testing.py +207 -0
- data/vendor/pygments-main/pygments/lexers/text.py +26 -0
- data/vendor/pygments-main/pygments/lexers/textedit.py +169 -0
- data/vendor/pygments-main/pygments/lexers/textfmts.py +430 -0
- data/vendor/pygments-main/pygments/lexers/theorem.py +474 -0
- data/vendor/pygments-main/pygments/lexers/tnt.py +263 -0
- data/vendor/pygments-main/pygments/lexers/trafficscript.py +54 -0
- data/vendor/pygments-main/pygments/lexers/typoscript.py +219 -0
- data/vendor/pygments-main/pygments/lexers/unicon.py +412 -0
- data/vendor/pygments-main/pygments/lexers/urbi.py +146 -0
- data/vendor/pygments-main/pygments/lexers/usd.py +90 -0
- data/vendor/pygments-main/pygments/lexers/varnish.py +190 -0
- data/vendor/pygments-main/pygments/lexers/verification.py +114 -0
- data/vendor/pygments-main/pygments/lexers/web.py +24 -0
- data/vendor/pygments-main/pygments/lexers/webidl.py +299 -0
- data/vendor/pygments-main/pygments/lexers/webmisc.py +991 -0
- data/vendor/pygments-main/pygments/lexers/whiley.py +116 -0
- data/vendor/pygments-main/pygments/lexers/x10.py +69 -0
- data/vendor/pygments-main/pygments/lexers/xorg.py +37 -0
- data/vendor/pygments-main/pygments/lexers/yang.py +104 -0
- data/vendor/pygments-main/pygments/lexers/zig.py +124 -0
- data/vendor/pygments-main/pygments/modeline.py +1 -1
- data/vendor/pygments-main/pygments/plugin.py +4 -2
- data/vendor/pygments-main/pygments/regexopt.py +1 -1
- data/vendor/pygments-main/pygments/scanner.py +2 -2
- data/vendor/pygments-main/pygments/sphinxext.py +2 -4
- data/vendor/pygments-main/pygments/style.py +61 -24
- data/vendor/pygments-main/pygments/styles/__init__.py +10 -4
- data/vendor/pygments-main/pygments/styles/abap.py +1 -1
- data/vendor/pygments-main/pygments/styles/algol.py +1 -1
- data/vendor/pygments-main/pygments/styles/algol_nu.py +1 -1
- data/vendor/pygments-main/pygments/styles/arduino.py +2 -2
- data/vendor/pygments-main/pygments/styles/autumn.py +1 -1
- data/vendor/pygments-main/pygments/styles/borland.py +1 -1
- data/vendor/pygments-main/pygments/styles/bw.py +1 -1
- data/vendor/pygments-main/pygments/styles/colorful.py +1 -1
- data/vendor/pygments-main/pygments/styles/default.py +1 -1
- data/vendor/pygments-main/pygments/styles/emacs.py +1 -1
- data/vendor/pygments-main/pygments/styles/friendly.py +1 -1
- data/vendor/pygments-main/pygments/styles/fruity.py +1 -1
- data/vendor/pygments-main/pygments/styles/igor.py +1 -1
- data/vendor/pygments-main/pygments/styles/inkpot.py +67 -0
- data/vendor/pygments-main/pygments/styles/lovelace.py +1 -1
- data/vendor/pygments-main/pygments/styles/manni.py +1 -1
- data/vendor/pygments-main/pygments/styles/monokai.py +4 -3
- data/vendor/pygments-main/pygments/styles/murphy.py +1 -1
- data/vendor/pygments-main/pygments/styles/native.py +1 -1
- data/vendor/pygments-main/pygments/styles/paraiso_dark.py +1 -1
- data/vendor/pygments-main/pygments/styles/paraiso_light.py +1 -1
- data/vendor/pygments-main/pygments/styles/pastie.py +1 -1
- data/vendor/pygments-main/pygments/styles/perldoc.py +1 -1
- data/vendor/pygments-main/pygments/styles/rainbow_dash.py +1 -1
- data/vendor/pygments-main/pygments/styles/rrt.py +1 -1
- data/vendor/pygments-main/pygments/styles/sas.py +1 -1
- data/vendor/pygments-main/pygments/styles/solarized.py +134 -0
- data/vendor/pygments-main/pygments/styles/stata_dark.py +41 -0
- data/vendor/pygments-main/pygments/styles/{stata.py → stata_light.py} +14 -15
- data/vendor/pygments-main/pygments/styles/tango.py +1 -1
- data/vendor/pygments-main/pygments/styles/trac.py +1 -1
- data/vendor/pygments-main/pygments/styles/vim.py +1 -1
- data/vendor/pygments-main/pygments/styles/vs.py +1 -1
- data/vendor/pygments-main/pygments/styles/xcode.py +1 -1
- data/vendor/pygments-main/pygments/token.py +1 -1
- data/vendor/pygments-main/pygments/unistring.py +47 -108
- data/vendor/pygments-main/pygments/util.py +15 -92
- metadata +69 -136
- data/CHANGELOG.md +0 -111
- data/README.md +0 -121
- data/circle.yml +0 -20
- data/test/test_data.py +0 -514
- data/test/test_data_generated +0 -2582
- data/vendor/custom_lexers/github.py +0 -565
- data/vendor/pygments-main/CHANGES +0 -1186
- data/vendor/pygments-main/MANIFEST.in +0 -6
- data/vendor/pygments-main/Makefile +0 -65
- data/vendor/pygments-main/README.rst +0 -39
- data/vendor/pygments-main/REVISION +0 -1
- data/vendor/pygments-main/TODO +0 -12
- data/vendor/pygments-main/doc/Makefile +0 -153
- data/vendor/pygments-main/doc/_static/favicon.ico +0 -0
- data/vendor/pygments-main/doc/_static/logo_new.png +0 -0
- data/vendor/pygments-main/doc/_static/logo_only.png +0 -0
- data/vendor/pygments-main/doc/_templates/docssidebar.html +0 -3
- data/vendor/pygments-main/doc/_templates/indexsidebar.html +0 -25
- data/vendor/pygments-main/doc/_themes/pygments14/layout.html +0 -98
- data/vendor/pygments-main/doc/_themes/pygments14/static/bodybg.png +0 -0
- data/vendor/pygments-main/doc/_themes/pygments14/static/docbg.png +0 -0
- data/vendor/pygments-main/doc/_themes/pygments14/static/listitem.png +0 -0
- data/vendor/pygments-main/doc/_themes/pygments14/static/logo.png +0 -0
- data/vendor/pygments-main/doc/_themes/pygments14/static/pocoo.png +0 -0
- data/vendor/pygments-main/doc/_themes/pygments14/static/pygments14.css_t +0 -401
- data/vendor/pygments-main/doc/_themes/pygments14/theme.conf +0 -15
- data/vendor/pygments-main/doc/conf.py +0 -241
- data/vendor/pygments-main/doc/docs/api.rst +0 -354
- data/vendor/pygments-main/doc/docs/authors.rst +0 -4
- data/vendor/pygments-main/doc/docs/changelog.rst +0 -1
- data/vendor/pygments-main/doc/docs/cmdline.rst +0 -166
- data/vendor/pygments-main/doc/docs/filterdevelopment.rst +0 -71
- data/vendor/pygments-main/doc/docs/filters.rst +0 -41
- data/vendor/pygments-main/doc/docs/formatterdevelopment.rst +0 -169
- data/vendor/pygments-main/doc/docs/formatters.rst +0 -48
- data/vendor/pygments-main/doc/docs/index.rst +0 -66
- data/vendor/pygments-main/doc/docs/integrate.rst +0 -40
- data/vendor/pygments-main/doc/docs/java.rst +0 -70
- data/vendor/pygments-main/doc/docs/lexerdevelopment.rst +0 -728
- data/vendor/pygments-main/doc/docs/lexers.rst +0 -69
- data/vendor/pygments-main/doc/docs/moinmoin.rst +0 -39
- data/vendor/pygments-main/doc/docs/plugins.rst +0 -93
- data/vendor/pygments-main/doc/docs/quickstart.rst +0 -205
- data/vendor/pygments-main/doc/docs/rstdirective.rst +0 -22
- data/vendor/pygments-main/doc/docs/styles.rst +0 -201
- data/vendor/pygments-main/doc/docs/tokens.rst +0 -372
- data/vendor/pygments-main/doc/docs/unicode.rst +0 -58
- data/vendor/pygments-main/doc/download.rst +0 -41
- data/vendor/pygments-main/doc/faq.rst +0 -139
- data/vendor/pygments-main/doc/index.rst +0 -54
- data/vendor/pygments-main/doc/languages.rst +0 -154
- data/vendor/pygments-main/doc/make.bat +0 -190
- data/vendor/pygments-main/doc/pygmentize.1 +0 -94
- data/vendor/pygments-main/external/autopygmentize +0 -101
- data/vendor/pygments-main/external/lasso-builtins-generator-9.lasso +0 -162
- data/vendor/pygments-main/external/markdown-processor.py +0 -67
- data/vendor/pygments-main/external/moin-parser.py +0 -112
- data/vendor/pygments-main/external/pygments.bashcomp +0 -38
- data/vendor/pygments-main/external/rst-directive.py +0 -82
- data/vendor/pygments-main/pygmentize +0 -8
- data/vendor/pygments-main/requirements.txt +0 -5
- data/vendor/pygments-main/scripts/check_sources.py +0 -211
- data/vendor/pygments-main/scripts/debug_lexer.py +0 -246
- data/vendor/pygments-main/scripts/detect_missing_analyse_text.py +0 -33
- data/vendor/pygments-main/scripts/epydoc.css +0 -280
- data/vendor/pygments-main/scripts/get_vimkw.py +0 -74
- data/vendor/pygments-main/scripts/pylintrc +0 -301
- data/vendor/pygments-main/scripts/vim2pygments.py +0 -935
- data/vendor/pygments-main/setup.cfg +0 -10
- data/vendor/pygments-main/setup.py +0 -77
- data/vendor/pygments-main/tox.ini +0 -7
- data/vendor/simplejson/.gitignore +0 -10
- data/vendor/simplejson/.travis.yml +0 -5
- data/vendor/simplejson/CHANGES.txt +0 -291
- data/vendor/simplejson/LICENSE.txt +0 -19
- data/vendor/simplejson/MANIFEST.in +0 -5
- data/vendor/simplejson/README.rst +0 -19
- data/vendor/simplejson/conf.py +0 -179
- data/vendor/simplejson/index.rst +0 -628
- data/vendor/simplejson/scripts/make_docs.py +0 -18
- data/vendor/simplejson/setup.py +0 -104
- data/vendor/simplejson/simplejson/__init__.py +0 -510
- data/vendor/simplejson/simplejson/_speedups.c +0 -2745
- data/vendor/simplejson/simplejson/decoder.py +0 -425
- data/vendor/simplejson/simplejson/encoder.py +0 -567
- data/vendor/simplejson/simplejson/ordered_dict.py +0 -119
- data/vendor/simplejson/simplejson/scanner.py +0 -77
- data/vendor/simplejson/simplejson/tests/__init__.py +0 -67
- data/vendor/simplejson/simplejson/tests/test_bigint_as_string.py +0 -55
- data/vendor/simplejson/simplejson/tests/test_check_circular.py +0 -30
- data/vendor/simplejson/simplejson/tests/test_decimal.py +0 -66
- data/vendor/simplejson/simplejson/tests/test_decode.py +0 -83
- data/vendor/simplejson/simplejson/tests/test_default.py +0 -9
- data/vendor/simplejson/simplejson/tests/test_dump.py +0 -67
- data/vendor/simplejson/simplejson/tests/test_encode_basestring_ascii.py +0 -46
- data/vendor/simplejson/simplejson/tests/test_encode_for_html.py +0 -32
- data/vendor/simplejson/simplejson/tests/test_errors.py +0 -34
- data/vendor/simplejson/simplejson/tests/test_fail.py +0 -91
- data/vendor/simplejson/simplejson/tests/test_float.py +0 -19
- data/vendor/simplejson/simplejson/tests/test_indent.py +0 -86
- data/vendor/simplejson/simplejson/tests/test_item_sort_key.py +0 -20
- data/vendor/simplejson/simplejson/tests/test_namedtuple.py +0 -121
- data/vendor/simplejson/simplejson/tests/test_pass1.py +0 -76
- data/vendor/simplejson/simplejson/tests/test_pass2.py +0 -14
- data/vendor/simplejson/simplejson/tests/test_pass3.py +0 -20
- data/vendor/simplejson/simplejson/tests/test_recursion.py +0 -67
- data/vendor/simplejson/simplejson/tests/test_scanstring.py +0 -117
- data/vendor/simplejson/simplejson/tests/test_separators.py +0 -42
- data/vendor/simplejson/simplejson/tests/test_speedups.py +0 -20
- data/vendor/simplejson/simplejson/tests/test_tuple.py +0 -49
- data/vendor/simplejson/simplejson/tests/test_unicode.py +0 -109
- data/vendor/simplejson/simplejson/tool.py +0 -39
@@ -1,38 +0,0 @@
|
|
1
|
-
#!bash
|
2
|
-
#
|
3
|
-
# Bash completion support for Pygments (the 'pygmentize' command).
|
4
|
-
#
|
5
|
-
|
6
|
-
_pygmentize()
|
7
|
-
{
|
8
|
-
local cur prev
|
9
|
-
|
10
|
-
COMPREPLY=()
|
11
|
-
cur=`_get_cword`
|
12
|
-
prev=${COMP_WORDS[COMP_CWORD-1]}
|
13
|
-
|
14
|
-
case "$prev" in
|
15
|
-
-f)
|
16
|
-
FORMATTERS=`pygmentize -L formatters | grep '* ' | cut -c3- | sed -e 's/,//g' -e 's/:$//'`
|
17
|
-
COMPREPLY=( $( compgen -W '$FORMATTERS' -- "$cur" ) )
|
18
|
-
return 0
|
19
|
-
;;
|
20
|
-
-l)
|
21
|
-
LEXERS=`pygmentize -L lexers | grep '* ' | cut -c3- | sed -e 's/,//g' -e 's/:$//'`
|
22
|
-
COMPREPLY=( $( compgen -W '$LEXERS' -- "$cur" ) )
|
23
|
-
return 0
|
24
|
-
;;
|
25
|
-
-S)
|
26
|
-
STYLES=`pygmentize -L styles | grep '* ' | cut -c3- | sed s/:$//`
|
27
|
-
COMPREPLY=( $( compgen -W '$STYLES' -- "$cur" ) )
|
28
|
-
return 0
|
29
|
-
;;
|
30
|
-
esac
|
31
|
-
|
32
|
-
if [[ "$cur" == -* ]]; then
|
33
|
-
COMPREPLY=( $( compgen -W '-f -l -S -L -g -O -P -F \
|
34
|
-
-N -H -h -V -o' -- "$cur" ) )
|
35
|
-
return 0
|
36
|
-
fi
|
37
|
-
}
|
38
|
-
complete -F _pygmentize -o default pygmentize
|
@@ -1,82 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
"""
|
3
|
-
The Pygments reStructuredText directive
|
4
|
-
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
5
|
-
|
6
|
-
This fragment is a Docutils_ 0.5 directive that renders source code
|
7
|
-
(to HTML only, currently) via Pygments.
|
8
|
-
|
9
|
-
To use it, adjust the options below and copy the code into a module
|
10
|
-
that you import on initialization. The code then automatically
|
11
|
-
registers a ``sourcecode`` directive that you can use instead of
|
12
|
-
normal code blocks like this::
|
13
|
-
|
14
|
-
.. sourcecode:: python
|
15
|
-
|
16
|
-
My code goes here.
|
17
|
-
|
18
|
-
If you want to have different code styles, e.g. one with line numbers
|
19
|
-
and one without, add formatters with their names in the VARIANTS dict
|
20
|
-
below. You can invoke them instead of the DEFAULT one by using a
|
21
|
-
directive option::
|
22
|
-
|
23
|
-
.. sourcecode:: python
|
24
|
-
:linenos:
|
25
|
-
|
26
|
-
My code goes here.
|
27
|
-
|
28
|
-
Look at the `directive documentation`_ to get all the gory details.
|
29
|
-
|
30
|
-
.. _Docutils: http://docutils.sf.net/
|
31
|
-
.. _directive documentation:
|
32
|
-
http://docutils.sourceforge.net/docs/howto/rst-directives.html
|
33
|
-
|
34
|
-
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
|
35
|
-
:license: BSD, see LICENSE for details.
|
36
|
-
"""
|
37
|
-
|
38
|
-
# Options
|
39
|
-
# ~~~~~~~
|
40
|
-
|
41
|
-
# Set to True if you want inline CSS styles instead of classes
|
42
|
-
INLINESTYLES = False
|
43
|
-
|
44
|
-
from pygments.formatters import HtmlFormatter
|
45
|
-
|
46
|
-
# The default formatter
|
47
|
-
DEFAULT = HtmlFormatter(noclasses=INLINESTYLES)
|
48
|
-
|
49
|
-
# Add name -> formatter pairs for every variant you want to use
|
50
|
-
VARIANTS = {
|
51
|
-
# 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True),
|
52
|
-
}
|
53
|
-
|
54
|
-
|
55
|
-
from docutils import nodes
|
56
|
-
from docutils.parsers.rst import directives, Directive
|
57
|
-
|
58
|
-
from pygments import highlight
|
59
|
-
from pygments.lexers import get_lexer_by_name, TextLexer
|
60
|
-
|
61
|
-
class Pygments(Directive):
|
62
|
-
""" Source code syntax hightlighting.
|
63
|
-
"""
|
64
|
-
required_arguments = 1
|
65
|
-
optional_arguments = 0
|
66
|
-
final_argument_whitespace = True
|
67
|
-
option_spec = dict([(key, directives.flag) for key in VARIANTS])
|
68
|
-
has_content = True
|
69
|
-
|
70
|
-
def run(self):
|
71
|
-
self.assert_has_content()
|
72
|
-
try:
|
73
|
-
lexer = get_lexer_by_name(self.arguments[0])
|
74
|
-
except ValueError:
|
75
|
-
# no lexer found - use the text one instead of an exception
|
76
|
-
lexer = TextLexer()
|
77
|
-
# take an arbitrary option if more than one is given
|
78
|
-
formatter = self.options and VARIANTS[list(self.options)[0]] or DEFAULT
|
79
|
-
parsed = highlight(u'\n'.join(self.content), lexer, formatter)
|
80
|
-
return [nodes.raw('', parsed, format='html')]
|
81
|
-
|
82
|
-
directives.register_directive('sourcecode', Pygments)
|
@@ -1,211 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python
|
2
|
-
# -*- coding: utf-8 -*-
|
3
|
-
"""
|
4
|
-
Checker for file headers
|
5
|
-
~~~~~~~~~~~~~~~~~~~~~~~~
|
6
|
-
|
7
|
-
Make sure each Python file has a correct file header
|
8
|
-
including copyright and license information.
|
9
|
-
|
10
|
-
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
|
11
|
-
:license: BSD, see LICENSE for details.
|
12
|
-
"""
|
13
|
-
|
14
|
-
from __future__ import print_function
|
15
|
-
|
16
|
-
import io
|
17
|
-
import os
|
18
|
-
import re
|
19
|
-
import sys
|
20
|
-
import getopt
|
21
|
-
from os.path import join, splitext, abspath
|
22
|
-
|
23
|
-
|
24
|
-
checkers = {}
|
25
|
-
|
26
|
-
|
27
|
-
def checker(*suffixes, **kwds):
|
28
|
-
only_pkg = kwds.pop('only_pkg', False)
|
29
|
-
|
30
|
-
def deco(func):
|
31
|
-
for suffix in suffixes:
|
32
|
-
checkers.setdefault(suffix, []).append(func)
|
33
|
-
func.only_pkg = only_pkg
|
34
|
-
return func
|
35
|
-
return deco
|
36
|
-
|
37
|
-
|
38
|
-
name_mail_re = r'[\w ]+(<.*?>)?'
|
39
|
-
copyright_re = re.compile(r'^ :copyright: Copyright 2006-2017 by '
|
40
|
-
r'the Pygments team, see AUTHORS\.$', re.UNICODE)
|
41
|
-
copyright_2_re = re.compile(r'^ %s(, %s)*[,.]$' %
|
42
|
-
(name_mail_re, name_mail_re), re.UNICODE)
|
43
|
-
is_const_re = re.compile(r'if.*?==\s+(None|False|True)\b')
|
44
|
-
|
45
|
-
misspellings = ["developement", "adress", "verificate", # ALLOW-MISSPELLING
|
46
|
-
"informations", "unlexer"] # ALLOW-MISSPELLING
|
47
|
-
|
48
|
-
|
49
|
-
@checker('.py')
|
50
|
-
def check_syntax(fn, lines):
|
51
|
-
if '#!/' in lines[0]:
|
52
|
-
lines = lines[1:]
|
53
|
-
if 'coding:' in lines[0]:
|
54
|
-
lines = lines[1:]
|
55
|
-
try:
|
56
|
-
compile('\n'.join(lines), fn, "exec")
|
57
|
-
except SyntaxError as err:
|
58
|
-
yield 0, "not compilable: %s" % err
|
59
|
-
|
60
|
-
|
61
|
-
@checker('.py')
|
62
|
-
def check_style_and_encoding(fn, lines):
|
63
|
-
for lno, line in enumerate(lines):
|
64
|
-
if len(line) > 110:
|
65
|
-
yield lno+1, "line too long"
|
66
|
-
if is_const_re.search(line):
|
67
|
-
yield lno+1, 'using == None/True/False'
|
68
|
-
|
69
|
-
|
70
|
-
@checker('.py', only_pkg=True)
|
71
|
-
def check_fileheader(fn, lines):
|
72
|
-
# line number correction
|
73
|
-
c = 1
|
74
|
-
if lines[0:1] == ['#!/usr/bin/env python']:
|
75
|
-
lines = lines[1:]
|
76
|
-
c = 2
|
77
|
-
|
78
|
-
llist = []
|
79
|
-
docopen = False
|
80
|
-
for lno, l in enumerate(lines):
|
81
|
-
llist.append(l)
|
82
|
-
if lno == 0:
|
83
|
-
if l != '# -*- coding: utf-8 -*-':
|
84
|
-
yield 1, "missing coding declaration"
|
85
|
-
elif lno == 1:
|
86
|
-
if l != '"""' and l != 'r"""':
|
87
|
-
yield 2, 'missing docstring begin (""")'
|
88
|
-
else:
|
89
|
-
docopen = True
|
90
|
-
elif docopen:
|
91
|
-
if l == '"""':
|
92
|
-
# end of docstring
|
93
|
-
if lno <= 4:
|
94
|
-
yield lno+c, "missing module name in docstring"
|
95
|
-
break
|
96
|
-
|
97
|
-
if l != "" and l[:4] != ' ' and docopen:
|
98
|
-
yield lno+c, "missing correct docstring indentation"
|
99
|
-
|
100
|
-
if lno == 2:
|
101
|
-
# if not in package, don't check the module name
|
102
|
-
modname = fn[:-3].replace('/', '.').replace('.__init__', '')
|
103
|
-
while modname:
|
104
|
-
if l.lower()[4:] == modname:
|
105
|
-
break
|
106
|
-
modname = '.'.join(modname.split('.')[1:])
|
107
|
-
else:
|
108
|
-
yield 3, "wrong module name in docstring heading"
|
109
|
-
modnamelen = len(l.strip())
|
110
|
-
elif lno == 3:
|
111
|
-
if l.strip() != modnamelen * "~":
|
112
|
-
yield 4, "wrong module name underline, should be ~~~...~"
|
113
|
-
|
114
|
-
else:
|
115
|
-
yield 0, "missing end and/or start of docstring..."
|
116
|
-
|
117
|
-
# check for copyright and license fields
|
118
|
-
license = llist[-2:-1]
|
119
|
-
if license != [" :license: BSD, see LICENSE for details."]:
|
120
|
-
yield 0, "no correct license info"
|
121
|
-
|
122
|
-
ci = -3
|
123
|
-
copyright = llist[ci:ci+1]
|
124
|
-
while copyright and copyright_2_re.match(copyright[0]):
|
125
|
-
ci -= 1
|
126
|
-
copyright = llist[ci:ci+1]
|
127
|
-
if not copyright or not copyright_re.match(copyright[0]):
|
128
|
-
yield 0, "no correct copyright info"
|
129
|
-
|
130
|
-
|
131
|
-
def main(argv):
|
132
|
-
try:
|
133
|
-
gopts, args = getopt.getopt(argv[1:], "vi:")
|
134
|
-
except getopt.GetoptError:
|
135
|
-
print("Usage: %s [-v] [-i ignorepath]* [path]" % argv[0])
|
136
|
-
return 2
|
137
|
-
opts = {}
|
138
|
-
for opt, val in gopts:
|
139
|
-
if opt == '-i':
|
140
|
-
val = abspath(val)
|
141
|
-
opts.setdefault(opt, []).append(val)
|
142
|
-
|
143
|
-
if len(args) == 0:
|
144
|
-
path = '.'
|
145
|
-
elif len(args) == 1:
|
146
|
-
path = args[0]
|
147
|
-
else:
|
148
|
-
print("Usage: %s [-v] [-i ignorepath]* [path]" % argv[0])
|
149
|
-
return 2
|
150
|
-
|
151
|
-
verbose = '-v' in opts
|
152
|
-
|
153
|
-
num = 0
|
154
|
-
out = io.StringIO()
|
155
|
-
|
156
|
-
# TODO: replace os.walk run with iteration over output of
|
157
|
-
# `svn list -R`.
|
158
|
-
|
159
|
-
for root, dirs, files in os.walk(path):
|
160
|
-
if '.hg' in dirs:
|
161
|
-
dirs.remove('.hg')
|
162
|
-
if 'examplefiles' in dirs:
|
163
|
-
dirs.remove('examplefiles')
|
164
|
-
if '-i' in opts and abspath(root) in opts['-i']:
|
165
|
-
del dirs[:]
|
166
|
-
continue
|
167
|
-
# XXX: awkward: for the Makefile call: don't check non-package
|
168
|
-
# files for file headers
|
169
|
-
in_pygments_pkg = root.startswith('./pygments')
|
170
|
-
for fn in files:
|
171
|
-
|
172
|
-
fn = join(root, fn)
|
173
|
-
if fn[:2] == './':
|
174
|
-
fn = fn[2:]
|
175
|
-
|
176
|
-
if '-i' in opts and abspath(fn) in opts['-i']:
|
177
|
-
continue
|
178
|
-
|
179
|
-
ext = splitext(fn)[1]
|
180
|
-
checkerlist = checkers.get(ext, None)
|
181
|
-
if not checkerlist:
|
182
|
-
continue
|
183
|
-
|
184
|
-
if verbose:
|
185
|
-
print("Checking %s..." % fn)
|
186
|
-
|
187
|
-
try:
|
188
|
-
lines = open(fn, 'rb').read().decode('utf-8').splitlines()
|
189
|
-
except (IOError, OSError) as err:
|
190
|
-
print("%s: cannot open: %s" % (fn, err))
|
191
|
-
num += 1
|
192
|
-
continue
|
193
|
-
|
194
|
-
for checker in checkerlist:
|
195
|
-
if not in_pygments_pkg and checker.only_pkg:
|
196
|
-
continue
|
197
|
-
for lno, msg in checker(fn, lines):
|
198
|
-
print(u"%s:%d: %s" % (fn, lno, msg), file=out)
|
199
|
-
num += 1
|
200
|
-
if verbose:
|
201
|
-
print()
|
202
|
-
if num == 0:
|
203
|
-
print("No errors found.")
|
204
|
-
else:
|
205
|
-
print(out.getvalue().rstrip('\n'))
|
206
|
-
print("%d error%s found." % (num, num > 1 and "s" or ""))
|
207
|
-
return int(num > 0)
|
208
|
-
|
209
|
-
|
210
|
-
if __name__ == '__main__':
|
211
|
-
sys.exit(main(sys.argv))
|
@@ -1,246 +0,0 @@
|
|
1
|
-
#!/usr/bin/python
|
2
|
-
# -*- coding: utf-8 -*-
|
3
|
-
"""
|
4
|
-
Lexing error finder
|
5
|
-
~~~~~~~~~~~~~~~~~~~
|
6
|
-
|
7
|
-
For the source files given on the command line, display
|
8
|
-
the text where Error tokens are being generated, along
|
9
|
-
with some context.
|
10
|
-
|
11
|
-
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
|
12
|
-
:license: BSD, see LICENSE for details.
|
13
|
-
"""
|
14
|
-
|
15
|
-
from __future__ import print_function
|
16
|
-
|
17
|
-
import os
|
18
|
-
import sys
|
19
|
-
|
20
|
-
# always prefer Pygments from source if exists
|
21
|
-
srcpath = os.path.join(os.path.dirname(__file__), '..')
|
22
|
-
if os.path.isdir(os.path.join(srcpath, 'pygments')):
|
23
|
-
sys.path.insert(0, srcpath)
|
24
|
-
|
25
|
-
|
26
|
-
from pygments.lexer import RegexLexer, ExtendedRegexLexer, LexerContext, \
|
27
|
-
ProfilingRegexLexer, ProfilingRegexLexerMeta
|
28
|
-
from pygments.lexers import get_lexer_by_name, find_lexer_class, \
|
29
|
-
find_lexer_class_for_filename
|
30
|
-
from pygments.token import Error, Text, _TokenType
|
31
|
-
from pygments.cmdline import _parse_options
|
32
|
-
|
33
|
-
|
34
|
-
class DebuggingRegexLexer(ExtendedRegexLexer):
|
35
|
-
"""Make the state stack, position and current match instance attributes."""
|
36
|
-
|
37
|
-
def get_tokens_unprocessed(self, text, stack=('root',)):
|
38
|
-
"""
|
39
|
-
Split ``text`` into (tokentype, text) pairs.
|
40
|
-
|
41
|
-
``stack`` is the inital stack (default: ``['root']``)
|
42
|
-
"""
|
43
|
-
tokendefs = self._tokens
|
44
|
-
self.ctx = ctx = LexerContext(text, 0)
|
45
|
-
ctx.stack = list(stack)
|
46
|
-
statetokens = tokendefs[ctx.stack[-1]]
|
47
|
-
while 1:
|
48
|
-
for rexmatch, action, new_state in statetokens:
|
49
|
-
self.m = m = rexmatch(text, ctx.pos, ctx.end)
|
50
|
-
if m:
|
51
|
-
if action is not None:
|
52
|
-
if type(action) is _TokenType:
|
53
|
-
yield ctx.pos, action, m.group()
|
54
|
-
ctx.pos = m.end()
|
55
|
-
else:
|
56
|
-
if not isinstance(self, ExtendedRegexLexer):
|
57
|
-
for item in action(self, m):
|
58
|
-
yield item
|
59
|
-
ctx.pos = m.end()
|
60
|
-
else:
|
61
|
-
for item in action(self, m, ctx):
|
62
|
-
yield item
|
63
|
-
if not new_state:
|
64
|
-
# altered the state stack?
|
65
|
-
statetokens = tokendefs[ctx.stack[-1]]
|
66
|
-
if new_state is not None:
|
67
|
-
# state transition
|
68
|
-
if isinstance(new_state, tuple):
|
69
|
-
for state in new_state:
|
70
|
-
if state == '#pop':
|
71
|
-
ctx.stack.pop()
|
72
|
-
elif state == '#push':
|
73
|
-
ctx.stack.append(ctx.stack[-1])
|
74
|
-
else:
|
75
|
-
ctx.stack.append(state)
|
76
|
-
elif isinstance(new_state, int):
|
77
|
-
# pop
|
78
|
-
del ctx.stack[new_state:]
|
79
|
-
elif new_state == '#push':
|
80
|
-
ctx.stack.append(ctx.stack[-1])
|
81
|
-
else:
|
82
|
-
assert False, 'wrong state def: %r' % new_state
|
83
|
-
statetokens = tokendefs[ctx.stack[-1]]
|
84
|
-
break
|
85
|
-
else:
|
86
|
-
try:
|
87
|
-
if ctx.pos >= ctx.end:
|
88
|
-
break
|
89
|
-
if text[ctx.pos] == '\n':
|
90
|
-
# at EOL, reset state to 'root'
|
91
|
-
ctx.stack = ['root']
|
92
|
-
statetokens = tokendefs['root']
|
93
|
-
yield ctx.pos, Text, u'\n'
|
94
|
-
ctx.pos += 1
|
95
|
-
continue
|
96
|
-
yield ctx.pos, Error, text[ctx.pos]
|
97
|
-
ctx.pos += 1
|
98
|
-
except IndexError:
|
99
|
-
break
|
100
|
-
|
101
|
-
|
102
|
-
def main(fn, lexer=None, options={}):
|
103
|
-
if lexer is not None:
|
104
|
-
lxcls = get_lexer_by_name(lexer).__class__
|
105
|
-
else:
|
106
|
-
lxcls = find_lexer_class_for_filename(os.path.basename(fn))
|
107
|
-
if lxcls is None:
|
108
|
-
name, rest = fn.split('_', 1)
|
109
|
-
lxcls = find_lexer_class(name)
|
110
|
-
if lxcls is None:
|
111
|
-
raise AssertionError('no lexer found for file %r' % fn)
|
112
|
-
print('Using lexer: %s (%s.%s)' % (lxcls.name, lxcls.__module__,
|
113
|
-
lxcls.__name__))
|
114
|
-
debug_lexer = False
|
115
|
-
# if profile:
|
116
|
-
# # does not work for e.g. ExtendedRegexLexers
|
117
|
-
# if lxcls.__bases__ == (RegexLexer,):
|
118
|
-
# # yes we can! (change the metaclass)
|
119
|
-
# lxcls.__class__ = ProfilingRegexLexerMeta
|
120
|
-
# lxcls.__bases__ = (ProfilingRegexLexer,)
|
121
|
-
# lxcls._prof_sort_index = profsort
|
122
|
-
# else:
|
123
|
-
# if lxcls.__bases__ == (RegexLexer,):
|
124
|
-
# lxcls.__bases__ = (DebuggingRegexLexer,)
|
125
|
-
# debug_lexer = True
|
126
|
-
# elif lxcls.__bases__ == (DebuggingRegexLexer,):
|
127
|
-
# # already debugged before
|
128
|
-
# debug_lexer = True
|
129
|
-
# else:
|
130
|
-
# # HACK: ExtendedRegexLexer subclasses will only partially work here.
|
131
|
-
# lxcls.__bases__ = (DebuggingRegexLexer,)
|
132
|
-
# debug_lexer = True
|
133
|
-
|
134
|
-
lx = lxcls(**options)
|
135
|
-
lno = 1
|
136
|
-
if fn == '-':
|
137
|
-
text = sys.stdin.read()
|
138
|
-
else:
|
139
|
-
with open(fn, 'rb') as fp:
|
140
|
-
text = fp.read().decode('utf-8')
|
141
|
-
text = text.strip('\n') + '\n'
|
142
|
-
tokens = []
|
143
|
-
states = []
|
144
|
-
|
145
|
-
def show_token(tok, state):
|
146
|
-
reprs = list(map(repr, tok))
|
147
|
-
print(' ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0], end=' ')
|
148
|
-
if debug_lexer:
|
149
|
-
print(' ' + ' ' * (29-len(reprs[0])) + ' : '.join(state) if state else '', end=' ')
|
150
|
-
print()
|
151
|
-
|
152
|
-
for type, val in lx.get_tokens(text):
|
153
|
-
lno += val.count('\n')
|
154
|
-
if type == Error and not ignerror:
|
155
|
-
print('Error parsing', fn, 'on line', lno)
|
156
|
-
if not showall:
|
157
|
-
print('Previous tokens' + (debug_lexer and ' and states' or '') + ':')
|
158
|
-
for i in range(max(len(tokens) - num, 0), len(tokens)):
|
159
|
-
if debug_lexer:
|
160
|
-
show_token(tokens[i], states[i])
|
161
|
-
else:
|
162
|
-
show_token(tokens[i], None)
|
163
|
-
print('Error token:')
|
164
|
-
l = len(repr(val))
|
165
|
-
print(' ' + repr(val), end=' ')
|
166
|
-
if debug_lexer and hasattr(lx, 'ctx'):
|
167
|
-
print(' ' * (60-l) + ' : '.join(lx.ctx.stack), end=' ')
|
168
|
-
print()
|
169
|
-
print()
|
170
|
-
return 1
|
171
|
-
tokens.append((type, val))
|
172
|
-
if debug_lexer:
|
173
|
-
if hasattr(lx, 'ctx'):
|
174
|
-
states.append(lx.ctx.stack[:])
|
175
|
-
else:
|
176
|
-
states.append(None)
|
177
|
-
if showall:
|
178
|
-
show_token((type, val), states[-1] if debug_lexer else None)
|
179
|
-
return 0
|
180
|
-
|
181
|
-
|
182
|
-
def print_help():
|
183
|
-
print('''\
|
184
|
-
Pygments development helper to quickly debug lexers.
|
185
|
-
|
186
|
-
scripts/debug_lexer.py [options] file ...
|
187
|
-
|
188
|
-
Give one or more filenames to lex them and display possible error tokens
|
189
|
-
and/or profiling info. Files are assumed to be encoded in UTF-8.
|
190
|
-
|
191
|
-
Selecting lexer and options:
|
192
|
-
|
193
|
-
-l NAME use lexer named NAME (default is to guess from
|
194
|
-
the given filenames)
|
195
|
-
-O OPTIONSTR use lexer options parsed from OPTIONSTR
|
196
|
-
|
197
|
-
Debugging lexing errors:
|
198
|
-
|
199
|
-
-n N show the last N tokens on error
|
200
|
-
-a always show all lexed tokens (default is only
|
201
|
-
to show them when an error occurs)
|
202
|
-
-e do not stop on error tokens
|
203
|
-
|
204
|
-
Profiling:
|
205
|
-
|
206
|
-
-p use the ProfilingRegexLexer to profile regexes
|
207
|
-
instead of the debugging lexer
|
208
|
-
-s N sort profiling output by column N (default is
|
209
|
-
column 4, the time per call)
|
210
|
-
''')
|
211
|
-
|
212
|
-
num = 10
|
213
|
-
showall = False
|
214
|
-
ignerror = False
|
215
|
-
lexer = None
|
216
|
-
options = {}
|
217
|
-
profile = False
|
218
|
-
profsort = 4
|
219
|
-
|
220
|
-
if __name__ == '__main__':
|
221
|
-
import getopt
|
222
|
-
opts, args = getopt.getopt(sys.argv[1:], 'n:l:aepO:s:h')
|
223
|
-
for opt, val in opts:
|
224
|
-
if opt == '-n':
|
225
|
-
num = int(val)
|
226
|
-
elif opt == '-a':
|
227
|
-
showall = True
|
228
|
-
elif opt == '-e':
|
229
|
-
ignerror = True
|
230
|
-
elif opt == '-l':
|
231
|
-
lexer = val
|
232
|
-
elif opt == '-p':
|
233
|
-
profile = True
|
234
|
-
elif opt == '-s':
|
235
|
-
profsort = int(val)
|
236
|
-
elif opt == '-O':
|
237
|
-
options = _parse_options([val])
|
238
|
-
elif opt == '-h':
|
239
|
-
print_help()
|
240
|
-
sys.exit(0)
|
241
|
-
ret = 0
|
242
|
-
if not args:
|
243
|
-
print_help()
|
244
|
-
for f in args:
|
245
|
-
ret += main(f, lexer, options)
|
246
|
-
sys.exit(bool(ret))
|