pygments.rb 1.1.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8bc5149afa7c5c2d0c10d4c6aff7bc3baa070995
-  data.tar.gz: 082c9c329d7efa34eeef65e3418cf692651df776
+  metadata.gz: 68bbaf00ae347f4be0aa0915c3c91d3702fb1018
+  data.tar.gz: 3a35c538fdf8ceec9ad26f3ac86cb708e73f7109
 SHA512:
-  metadata.gz: 804642de1b753e1f29469248eba2595c2d0bc472336797688cfe73d8c835bc2d1e8da62651d4060feeae95a3cb48694eab67394210a952f6695f7ba41c17d67c
-  data.tar.gz: 0308d5e79197552f817854ad6827dfe0d8d8a500e34b67afd597381777149b9ae26b54ed86aedec3a68a87a9c948ca3238b7ed465bc35d9a65a3b68cb8355891
+  metadata.gz: 1ab74f4da1c783693f827b3245abf4a9d88c75fc92fbaeba78950f29638f36e85e61cef83d936537937e33749237d12b139ec46c1df3d03e8aa0a38cb629be0c
+  data.tar.gz: 0afd2f0409c4932453582ab242f8501601528a266411b6bcd1b1fc235da82074832a5d71d9adce682f0dcd099e23286e22342236c132dc9cd73b635b78878b25

data/CHANGELOG.md

@@ -1,7 +1,14 @@
-changelog
+CHANGELOG
 ===========
 
+Version 1.2.0 (2017/09/13)
+-----------------------------
+
+* Exclude symlinks from the gem package to solve Windows issues [#181](https://github.com/tmm1/pygments.rb/pull/181)
+* Upgrade pygments to 2.0.0 [#180](https://github.com/tmm1/pygments.rb/pull/180)
+
 Version 1.1.2 (2017/04/03)
+-----------------------------
 
 * Resolves #176 exclude find_error.py symlink from gem [#177](https://github.com/tmm1/pygments.rb/pull/177)
 

data/Rakefile

@@ -63,5 +63,6 @@ namespace :vendor do
     FileUtils.cd(LEXERS_DIR) { sh "python _mapping.py" }
   end
 
+  desc 'update vendor/pygments-main'
   task :update => [:clobber, 'vendor/pygments-main', :load_lexers]
 end

data/lib/pygments/version.rb

@@ -1,5 +1,5 @@
 # coding: utf-8
 # frozen_string_literal: true
 module Pygments
-  VERSION = '1.1.2'
+  VERSION = '1.2.0'
 end

data/pygments.rb.gemspec

@@ -21,6 +21,5 @@ Gem::Specification.new do |s|
   # s.extensions = ['ext/extconf.rb']
   s.require_paths = ['lib']
 
-  exclude = `find . -type l -printf '%P\\0'`.split("\0").map {|f| "':!#{f}'" } * ' '
-  s.files = `git ls-files -- . #{exclude}`.split("\n")
+  s.files = `git ls-files`.split("\n").select { |f| !File.symlink?(f) }
 end

data/vendor/pygments-main/AUTHORS

@@ -155,6 +155,7 @@ Other contributors, listed alphabetically, are:
 * Dominik Picheta -- Nimrod lexer
 * Andrew Pinkham -- RTF Formatter Refactoring
 * Clément Prévost -- UrbiScript lexer
+* Tanner Prynn -- cmdline -x option and loading lexers from files
 * Oleh Prypin -- Crystal lexer (based on Ruby lexer)
 * Elias Rabel -- Fortran fixed form lexer
 * raichoo -- Idris lexer
@@ -173,6 +174,7 @@ Other contributors, listed alphabetically, are:
 * Matteo Sasso -- Common Lisp lexer
 * Joe Schafer -- Ada lexer
 * Ken Schutte -- Matlab lexers
+* René Schwaiger -- Rainbow Dash style
 * Sebastian Schweizer -- Whiley lexer
 * Tassilo Schweyer -- Io, MOOCode lexers
 * Ted Shaw -- AutoIt lexer

data/vendor/pygments-main/CHANGES

@@ -6,9 +6,9 @@ Issue numbers refer to the tracker at
 pull request numbers to the requests at
 <https://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>.
 
-Version 2.2
------------
-(in development)
+Version 2.2.0
+-------------
+(release Jan 22, 2017)
 
 - Added lexers:
 
@@ -24,6 +24,17 @@ Version 2.2
   * NCAR command language (PR#536)
   * Extempore (PR#530)
   * Cap'n Proto (PR#595)
+  * Whiley (PR#573)
+  * Monte (PR#592)
+  * Crystal (PR#576)
+  * Snowball (PR#589)
+  * CapDL (PR#579)
+  * NuSMV (PR#564)
+  * SAS, Stata (PR#593)
+
+- Added the ability to load lexer and formatter classes directly from files
+  with the `-x` command line option and the `lexers.load_lexer_from_file()`
+  and `formatters.load_formatter_from_file()` functions. (PR#559)
 
 - Added `lexers.find_lexer_class_by_name()`. (#1203)
 
@@ -41,12 +52,24 @@ Version 2.2
 
 - Improved the CSS lexer. (#1083, #1130)
 
+- Added "Rainbow Dash" style. (PR#623)
+
+- Delay loading `pkg_resources`, which takes a long while to import. (PR#690)
+
+
+Version 2.1.3
+-------------
+(released Mar 2, 2016)
+
+- Fixed regression in Bash lexer (PR#563)
+
 
 Version 2.1.2
 -------------
-(in development)
+(released Feb 29, 2016)
 
 - Fixed Python 3 regression in image formatter (#1215)
+- Fixed regression in Bash lexer (PR#562)
 
 
 Version 2.1.1

data/vendor/pygments-main/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2006-2015 by the respective authors (see AUTHORS file).
+Copyright (c) 2006-2017 by the respective authors (see AUTHORS file).
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

data/vendor/pygments-main/MANIFEST.in

@@ -1,6 +1,6 @@
 include pygmentize
 include external/*
-include Makefile CHANGES LICENSE AUTHORS TODO ez_setup.py
+include Makefile CHANGES LICENSE AUTHORS TODO
 recursive-include tests *
 recursive-include doc *
 recursive-include scripts *

data/vendor/pygments-main/Makefile

@@ -4,7 +4,7 @@
 #
 # Combines scripts for common tasks.
 #
-# :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+# :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
 # :license: BSD, see LICENSE for details.
 #
 

data/vendor/pygments-main/REVISION

@@ -1 +1 @@
-a042025b350c
+7941677dc77d

data/vendor/pygments-main/doc/_themes/pygments14/layout.html

@@ -82,7 +82,7 @@
 
 {% block footer %}
     <div class="footer" role="contentinfo">
-      &copy; Copyright 2006-2015, Georg Brandl and Pygments contributors.
+      &copy; Copyright 2006-2017, Georg Brandl and Pygments contributors.
       Created using <a href="http://sphinx-doc.org/">Sphinx</a> {{
       sphinx_version }}. <br/>
       Pygments logo created by <a href="http://joelunger.com">Joel Unger</a>.

data/vendor/pygments-main/doc/_themes/pygments14/static/pygments14.css_t

@@ -4,7 +4,7 @@
  *
  * Sphinx stylesheet -- pygments14 theme.  Heavily copied from sphinx13.
  *
- * :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+ * :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */

data/vendor/pygments-main/doc/docs/api.rst

@@ -62,6 +62,18 @@ Functions from :mod:`pygments.lexers`:
     Will raise :exc:`pygments.util.ClassNotFound` if not lexer for that mimetype
     is found.
 
+.. function:: load_lexer_from_file(filename, lexername="CustomLexer", **options)
+
+    Return a `Lexer` subclass instance loaded from the provided file, relative
+    to the current directory. The file is expected to contain a Lexer class
+    named `lexername` (by default, CustomLexer). Users should be very careful with
+    the input, because this method is equivalent to running eval on the input file.
+    The lexer is given the `options` at its instantiation.
+
+    :exc:`ClassNotFound` is raised if there are any errors loading the Lexer
+
+    .. versionadded:: 2.2
+
 .. function:: guess_lexer(text, **options)
 
     Return a `Lexer` subclass instance that's guessed from the text in
@@ -125,6 +137,17 @@ Functions from :mod:`pygments.formatters`:
     Will raise :exc:`pygments.util.ClassNotFound` if no formatter for that filename
     is found.
 
+.. function:: load_formatter_from_file(filename, formattername="CustomFormatter", **options)
+
+    Return a `Formatter` subclass instance loaded from the provided file, relative
+    to the current directory. The file is expected to contain a Formatter class
+    named ``formattername`` (by default, CustomFormatter). Users should be very
+    careful with the input, because this method is equivalent to running eval
+    on the input file. The formatter is given the `options` at its instantiation.
+
+    :exc:`ClassNotFound` is raised if there are any errors loading the Formatter
+
+    .. versionadded:: 2.2
 
 .. module:: pygments.styles
 

data/vendor/pygments-main/doc/docs/cmdline.rst

@@ -99,6 +99,23 @@ The ``-N`` option guesses a lexer name for a given filename, so that ::
 will print out ``python``.  It won't highlight anything yet.  If no specific
 lexer is known for that filename, ``text`` is printed.
 
+Custom Lexers and Formatters
+----------------------------
+
+.. versionadded:: 2.2
+
+The ``-x`` flag enables custom lexers and formatters to be loaded
+from files relative to the current directory. Create a file with a class named
+CustomLexer or CustomFormatter, then specify it on the command line::
+
+    $ pygmentize -l your_lexer.py -f your_formatter.py -x
+
+You can also specify the name of your class with a colon::
+
+    $ pygmentize -l your_lexer.py:SomeLexer -x
+
+For more information, see :doc:`the Pygments documentation on Lexer development
+<lexerdevelopment>`.
 
 Getting help
 ------------

data/vendor/pygments-main/doc/docs/lexerdevelopment.rst

@@ -88,8 +88,47 @@ one.
 Adding and testing a new lexer
 ==============================
 
-Using a lexer that is not part of Pygments can be done via the Python API.  You
-can import and instantiate the lexer, and pass it to :func:`pygments.highlight`.
+The easiest way to use a new lexer is to use Pygments' support for loading
+the lexer from a file relative to your current directory.
+
+First, change the name of your lexer class to CustomLexer:
+
+.. code-block:: python
+
+    from pygments.lexer import RegexLexer
+    from pygments.token import *
+
+    class CustomLexer(RegexLexer):
+        """All your lexer code goes here!"""
+
+Then you can load the lexer from the command line with the additional
+flag ``-x``:
+
+.. code-block:: console
+
+    $ pygmentize -l your_lexer_file.py -x
+
+To specify a class name other than CustomLexer, append it with a colon:
+
+.. code-block:: console
+
+    $ pygmentize -l your_lexer.py:SomeLexer -x
+
+Or, using the Python API:
+
+.. code-block:: python
+
+    # For a lexer named CustomLexer
+    your_lexer = load_lexer_from_file(filename, **options)
+
+    # For a lexer named MyNewLexer
+    your_named_lexer = load_lexer_from_file(filename, "MyNewLexer", **options)
+
+When loading custom lexers and formatters, be extremely careful to use only
+trusted files; Pygments will perform the equivalent of ``eval`` on them.
+
+If you only want to use your lexer with the Pygments API, you can import and
+instantiate the lexer yourself, then pass it to :func:`pygments.highlight`.
 
 To prepare your new lexer for inclusion in the Pygments distribution, so that it
 will be found when passing filenames or lexer aliases from the command line, you
@@ -361,7 +400,7 @@ There are a few more things you can do with states:
           tokens = {...}
 
           def get_tokens_unprocessed(self, text, stack=('root', 'otherstate')):
-              for item in RegexLexer.get_tokens_unprocessed(text, stack):
+              for item in RegexLexer.get_tokens_unprocessed(self, text, stack):
                   yield item
 
   Some lexers like the `PhpLexer` use this to make the leading ``<?php``

data/vendor/pygments-main/external/autopygmentize

@@ -1,6 +1,6 @@
 #!/bin/bash
 # Best effort auto-pygmentization with transparent decompression
-# by Reuben Thomas 2008-2015
+# by Reuben Thomas 2008-2016
 # This program is in the public domain.
 
 # Strategy: first see if pygmentize can find a lexer; if not, ask file; if that finds nothing, fail
@@ -25,6 +25,7 @@ if [[ "$lexer" == text ]]; then
         text/x-awk) lexer=awk;;
         text/x-c) lexer=c;;
         text/x-c++) lexer=cpp;;
+        text/x-crystal) lexer=crystal;;
         text/x-diff) lexer=diff;;
         text/x-fortran) lexer=fortran;;
         text/x-gawk) lexer=gawk;;
@@ -40,7 +41,6 @@ if [[ "$lexer" == text ]]; then
         text/x-po) lexer=po;;
         text/x-python) lexer=python;;
         text/x-ruby) lexer=ruby;;
-        text/x-crystal) lexer=crystal;;
         text/x-shellscript) lexer=sh;;
         text/x-tcl) lexer=tcl;;
         text/x-tex|text/x-texinfo) lexer=latex;; # FIXME: texinfo really needs its own lexer
@@ -66,19 +66,36 @@ if [[ "$lexer" == text ]]; then
     esac
 fi
 
+# Find a preprocessor for compressed files
+concat=cat
+case $(file $file_common_opts --mime-type "$file") in
+    application/x-gzip)  concat=zcat;;
+    application/x-bzip2) concat=bzcat;;
+    application/x-xz)    concat=xzcat;;
+esac
+
+# Find a suitable lexer, preceded by a hex dump for binary files
+prereader=""
 encoding=$(file --mime-encoding --uncompress $file_common_opts "$file")
 if [[ $encoding == "binary" ]]; then
-    encoding="latin1"
+    prereader="od -x" # POSIX fallback
+    if [[ -n $(which hd) ]]; then
+        prereader="hd" # preferred
+    fi
+    lexer=hexdump
+    encoding=latin1
 fi
-
 if [[ -n "$lexer" ]]; then
-    concat=cat
-    case $(file $file_common_opts --mime-type "$file") in
-        application/x-gzip)  concat=zcat;;
-        application/x-bzip2) concat=bzcat;;
-        application/x-xz)    concat=xzcat;;
-    esac
-    exec $concat "$file" | pygmentize -O inencoding=$encoding $PYGMENTIZE_OPTS $options -l $lexer
+    reader="pygmentize -O inencoding=$encoding $PYGMENTIZE_OPTS $options -l $lexer"
+fi
+
+# If we found a reader, run it
+if [[ -n "$reader" ]]; then
+    if [[ -n "$prereader" ]]; then
+        exec $concat "$file" | $prereader | $reader
+    else
+        exec $concat "$file" | $reader
+    fi
 fi
 
 exit 1

data/vendor/pygments-main/external/markdown-processor.py

@@ -22,7 +22,7 @@
 
     .. _Markdown: https://pypi.python.org/pypi/Markdown
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/external/moin-parser.py

@@ -31,7 +31,7 @@
     If you do not want to do that and are willing to accept larger HTML
     output, you can set the INLINESTYLES option below to True.
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/external/rst-directive.py

@@ -31,7 +31,7 @@
     .. _directive documentation:
        http://docutils.sourceforge.net/docs/howto/rst-directives.html
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/__init__.py

@@ -22,14 +22,14 @@
     .. _Pygments tip:
        http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 import sys
 
 from pygments.util import StringIO, BytesIO
 
-__version__ = '2.2a0'
+__version__ = '2.2.0'
 __docformat__ = 'restructuredtext'
 
 __all__ = ['lex', 'format', 'highlight']

data/vendor/pygments-main/pygments/cmdline.py

@@ -5,12 +5,13 @@
 
     Command line interface.
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from __future__ import print_function
 
+import os
 import sys
 import getopt
 from textwrap import dedent
@@ -19,19 +20,20 @@ from pygments import __version__, highlight
 from pygments.util import ClassNotFound, OptionError, docstring_headline, \
     guess_decode, guess_decode_from_terminal, terminal_encoding
 from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
-    get_lexer_for_filename, find_lexer_class_for_filename
+    load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename
 from pygments.lexers.special import TextLexer
 from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
 from pygments.formatters import get_all_formatters, get_formatter_by_name, \
-    get_formatter_for_filename, find_formatter_class
+    load_formatter_from_file, get_formatter_for_filename, find_formatter_class
 from pygments.formatters.terminal import TerminalFormatter
+from pygments.formatters.terminal256 import Terminal256Formatter
 from pygments.filters import get_all_filters, find_filter_class
 from pygments.styles import get_all_styles, get_style_by_name
 
 
 USAGE = """\
 Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
-          [-O <options>] [-P <option=value>] [-s] [-v] [-o <outfile>] [<infile>]
+          [-O <options>] [-P <option=value>] [-s] [-v] [-x] [-o <outfile>] [<infile>]
 
        %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
        %s -L [<which> ...]
@@ -57,6 +59,14 @@ Likewise, <formatter> is a formatter name, and will be guessed from
 the extension of the output file name. If no output file is given,
 the terminal formatter will be used by default.
 
+The additional option -x allows custom lexers and formatters to be
+loaded from a .py file relative to the current working directory. For
+example, ``-l ./customlexer.py -x``. By default, this option expects a
+file with a class named CustomLexer or CustomFormatter; you can also
+specify your own class name with a colon (``-l ./lexer.py:MyLexer``).
+Users should be very careful not to use this option with untrusted files,
+because it will import and run them.
+
 With the -O option, you can give the lexer and formatter a comma-
 separated list of options, e.g. ``-O bg=light,python=cool``.
 
@@ -223,7 +233,7 @@ def main_inner(popts, args, usage):
         return 0
 
     if opts.pop('-V', None) is not None:
-        print('Pygments version %s, (c) 2006-2015 by Georg Brandl.' % __version__)
+        print('Pygments version %s, (c) 2006-2017 by Georg Brandl.' % __version__)
         return 0
 
     # handle ``pygmentize -L``
@@ -314,17 +324,35 @@ def main_inner(popts, args, usage):
     F_opts = _parse_filters(F_opts)
     opts.pop('-F', None)
 
+    allow_custom_lexer_formatter = False
+    # -x: allow custom (eXternal) lexers and formatters
+    if opts.pop('-x', None) is not None:
+        allow_custom_lexer_formatter = True
+
     # select lexer
     lexer = None
 
     # given by name?
     lexername = opts.pop('-l', None)
     if lexername:
-        try:
-            lexer = get_lexer_by_name(lexername, **parsed_opts)
-        except (OptionError, ClassNotFound) as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        # custom lexer, located relative to user's cwd
+        if allow_custom_lexer_formatter and '.py' in lexername:
+            try:
+                if ':' in lexername:
+                    filename, name = lexername.rsplit(':', 1)
+                    lexer = load_lexer_from_file(filename, name,
+                                                 **parsed_opts)
+                else:
+                    lexer = load_lexer_from_file(lexername, **parsed_opts)
+            except ClassNotFound as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        else:
+            try:
+                lexer = get_lexer_by_name(lexername, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
 
     # read input code
     code = None
@@ -401,11 +429,24 @@ def main_inner(popts, args, usage):
     outfn = opts.pop('-o', None)
     fmter = opts.pop('-f', None)
     if fmter:
-        try:
-            fmter = get_formatter_by_name(fmter, **parsed_opts)
-        except (OptionError, ClassNotFound) as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        # custom formatter, located relative to user's cwd
+        if allow_custom_lexer_formatter and '.py' in fmter:
+            try:
+                if ':' in fmter:
+                    file, fmtername = fmter.rsplit(':', 1)
+                    fmter = load_formatter_from_file(file, fmtername,
+                                                     **parsed_opts)
+                else:
+                    fmter = load_formatter_from_file(fmter, **parsed_opts)
+            except ClassNotFound as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        else:
+            try:
+                fmter = get_formatter_by_name(fmter, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
 
     if outfn:
         if not fmter:
@@ -421,7 +462,10 @@ def main_inner(popts, args, usage):
             return 1
     else:
         if not fmter:
-            fmter = TerminalFormatter(**parsed_opts)
+            if '256' in os.environ.get('TERM', ''):
+                fmter = Terminal256Formatter(**parsed_opts)
+            else:
+                fmter = TerminalFormatter(**parsed_opts)
         if sys.version_info > (3,):
             # Python 3: we have to use .buffer to get a binary stream
             outfile = sys.stdout.buffer
@@ -495,7 +539,7 @@ def main(args=sys.argv):
     usage = USAGE % ((args[0],) * 6)
 
     try:
-        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgs")
+        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgsx")
     except getopt.GetoptError:
         print(usage, file=sys.stderr)
         return 2