pygments.rb 0.5.4 → 0.6.0

data/vendor/pygments-main/pygments/lexers/text.py

@@ -5,7 +5,7 @@
 
     Lexers for non-source code file types.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -13,10 +13,11 @@ import re
 from bisect import bisect
 
 from pygments.lexer import Lexer, LexerContext, RegexLexer, ExtendedRegexLexer, \
-     bygroups, include, using, this, do_insertions
+     bygroups, include, using, this, do_insertions, default
 from pygments.token import Punctuation, Text, Comment, Keyword, Name, String, \
      Generic, Operator, Number, Whitespace, Literal
 from pygments.util import get_bool_opt, ClassNotFound
+from pygments.lexers.agile import PythonLexer
 from pygments.lexers.other import BashLexer
 
 __all__ = ['IniLexer', 'PropertiesLexer', 'SourcesListLexer', 'BaseMakefileLexer',
@@ -25,7 +26,8 @@ __all__ = ['IniLexer', 'PropertiesLexer', 'SourcesListLexer', 'BaseMakefileLexer
            'RstLexer', 'VimLexer', 'GettextLexer', 'SquidConfLexer',
            'DebianControlLexer', 'DarcsPatchLexer', 'YamlLexer',
            'LighttpdConfLexer', 'NginxConfLexer', 'CMakeLexer', 'HttpLexer',
-           'PyPyLogLexer', 'RegeditLexer', 'HxmlLexer', 'EbnfLexer']
+           'PyPyLogLexer', 'RegeditLexer', 'HxmlLexer', 'EbnfLexer',
+           'TodotxtLexer', 'DockerLexer']
 
 
 class IniLexer(RegexLexer):
@@ -61,7 +63,7 @@ class RegeditLexer(RegexLexer):
     <http://en.wikipedia.org/wiki/Windows_Registry#.REG_files>`_ files produced
     by regedit.
 
-    *New in Pygments 1.6.*
+    .. versionadded:: 1.6
     """
 
     name = 'reg'
@@ -102,7 +104,7 @@ class PropertiesLexer(RegexLexer):
     """
     Lexer for configuration files in Java's properties format.
 
-    *New in Pygments 1.4.*
+    .. versionadded:: 1.4
     """
 
     name = 'Properties'
@@ -124,7 +126,7 @@ class SourcesListLexer(RegexLexer):
     """
     Lexer that highlights debian sources.list files.
 
-    *New in Pygments 0.7.*
+    .. versionadded:: 0.7
     """
 
     name = 'Debian Sourcelist'
@@ -180,7 +182,7 @@ class MakefileLexer(Lexer):
 
     name = 'Makefile'
     aliases = ['make', 'makefile', 'mf', 'bsdmake']
-    filenames = ['*.mak', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile']
+    filenames = ['*.mak', '*.mk', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile']
     mimetypes = ['text/x-makefile']
 
     r_special = re.compile(r'^(?:'
@@ -207,12 +209,17 @@ class MakefileLexer(Lexer):
         for item in do_insertions(ins, lex.get_tokens_unprocessed(done)):
             yield item
 
+    def analyse_text(text):
+        # Many makefiles have $(BIG_CAPS) style variables
+        if re.search(r'\$\([A-Z_]+\)', text):
+            return 0.1
+
 
 class BaseMakefileLexer(RegexLexer):
     """
     Lexer for simple Makefiles (no preprocessing).
 
-    *New in Pygments 0.10.*
+    .. versionadded:: 0.10
     """
 
     name = 'Base Makefile'
@@ -222,15 +229,17 @@ class BaseMakefileLexer(RegexLexer):
 
     tokens = {
         'root': [
+            # recipes (need to allow spaces because of expandtabs)
             (r'^(?:[\t ]+.*\n|\n)+', using(BashLexer)),
-            (r'\$\((?:.*\\\n|.*\n)+', using(BashLexer)),
+            # special variables
+            (r'\$[<@$+%?|*]', Keyword),
             (r'\s+', Text),
             (r'#.*?\n', Comment),
-            (r'(export)(\s+)(?=[a-zA-Z0-9_${}\t -]+\n)',
+            (r'(export)(\s+)(?=[\w${}\t -]+\n)',
              bygroups(Keyword, Text), 'export'),
             (r'export\s+', Keyword),
             # assignment
-            (r'([a-zA-Z0-9_${}.-]+)(\s*)([!?:+]?=)([ \t]*)((?:.*\\\n)+|.*\n)',
+            (r'([\w${}.-]+)(\s*)([!?:+]?=)([ \t]*)((?:.*\\\n)+|.*\n)',
              bygroups(Name.Variable, Text, Operator, Text, using(BashLexer))),
             # strings
             (r'(?s)"(\\\\|\\.|[^"\\])*"', String.Double),
@@ -238,20 +247,29 @@ class BaseMakefileLexer(RegexLexer):
             # targets
             (r'([^\n:]+)(:+)([ \t]*)', bygroups(Name.Function, Operator, Text),
              'block-header'),
-            # TODO: add paren handling (grr)
+            # expansions
+            (r'\$\(', Keyword, 'expansion'),
+        ],
+        'expansion': [
+            (r'[^$a-zA-Z_)]+', Text),
+            (r'[a-zA-Z_]+', Name.Variable),
+            (r'\$', Keyword),
+            (r'\(', Keyword, '#push'),
+            (r'\)', Keyword, '#pop'),
         ],
         'export': [
-            (r'[a-zA-Z0-9_${}-]+', Name.Variable),
+            (r'[\w${}-]+', Name.Variable),
             (r'\n', Text, '#pop'),
             (r'\s+', Text),
         ],
         'block-header': [
-            (r'[^,\\\n#]+', Number),
-            (r',', Punctuation),
-            (r'#.*?\n', Comment),
+            (r'[,|]', Punctuation),
+            (r'#.*?\n', Comment, '#pop'),
             (r'\\\n', Text), # line continuation
-            (r'\\.', Text),
-            (r'(?:[\t ]+.*\n|\n)+', using(BashLexer), '#pop'),
+            (r'\$\(', Keyword, 'expansion'),
+            (r'[a-zA-Z_]+', Name),
+            (r'\n', Text, '#pop'),
+            (r'.', Text),
         ],
     }
 
@@ -297,7 +315,7 @@ class DarcsPatchLexer(RegexLexer):
     format.  Examples of this format are derived by commands such as
     ``darcs annotate --patch`` and ``darcs send``.
 
-    *New in Pygments 0.10.*
+    .. versionadded:: 0.10
     """
     name = 'Darcs Patch'
     aliases = ['dpatch']
@@ -410,7 +428,7 @@ class BBCodeLexer(RegexLexer):
     """
     A lexer that highlights BBCode(-like) syntax.
 
-    *New in Pygments 0.6.*
+    .. versionadded:: 0.6
     """
 
     name = 'BBCode'
@@ -485,7 +503,7 @@ class TexLexer(RegexLexer):
         'command': [
             (r'\[.*?\]', Name.Attribute),
             (r'\*', Keyword),
-            (r'', Text, '#pop'),
+            default('#pop'),
         ],
     }
 
@@ -501,7 +519,7 @@ class GroffLexer(RegexLexer):
     Lexer for the (g)roff typesetting language, supporting groff
     extensions. Mainly useful for highlighting manpage sources.
 
-    *New in Pygments 0.6.*
+    .. versionadded:: 0.6
     """
 
     name = 'Groff'
@@ -556,7 +574,7 @@ class ApacheConfLexer(RegexLexer):
     Lexer for configuration files following the Apache config file
     format.
 
-    *New in Pygments 0.6.*
+    .. versionadded:: 0.6
     """
 
     name = 'ApacheConf'
@@ -571,7 +589,7 @@ class ApacheConfLexer(RegexLexer):
             (r'(#.*?)$', Comment),
             (r'(<[^\s>]+)(?:(\s+)(.*?))?(>)',
              bygroups(Name.Tag, Text, String, Name.Tag)),
-            (r'([a-zA-Z][a-zA-Z0-9_]*)(\s+)',
+            (r'([a-z]\w*)(\s+)',
              bygroups(Name.Builtin, Text), 'value'),
             (r'\.+', Text),
         ],
@@ -580,7 +598,7 @@ class ApacheConfLexer(RegexLexer):
             (r'[^\S\n]+', Text),
             (r'\d+\.\d+\.\d+\.\d+(?:/\d+)?', Number),
             (r'\d+', Number),
-            (r'/([a-zA-Z0-9][a-zA-Z0-9_./-]+)', String.Other),
+            (r'/([a-z0-9][\w./-]+)', String.Other),
             (r'(on|off|none|any|all|double|email|dns|min|minimal|'
              r'os|productonly|full|emerg|alert|crit|error|warn|'
              r'notice|info|debug|registry|script|inetd|standalone|'
@@ -595,7 +613,7 @@ class MoinWikiLexer(RegexLexer):
     """
     For MoinMoin (and Trac) Wiki markup.
 
-    *New in Pygments 0.7.*
+    .. versionadded:: 0.7
     """
 
     name = 'MoinMoin/Trac Wiki markup'
@@ -640,14 +658,17 @@ class RstLexer(RegexLexer):
     """
     For `reStructuredText <http://docutils.sf.net/rst.html>`_ markup.
 
-    *New in Pygments 0.7.*
+    .. versionadded:: 0.7
 
     Additional options accepted:
 
     `handlecodeblocks`
-        Highlight the contents of ``.. sourcecode:: langauge`` and
-        ``.. code:: language`` directives with a lexer for the given
-        language (default: ``True``). *New in Pygments 0.8.*
+        Highlight the contents of ``.. sourcecode:: language``,
+        ``.. code:: language`` and ``.. code-block:: language``
+        directives with a lexer for the given language (default:
+        ``True``).
+
+        .. versionadded:: 0.8
     """
     name = 'reStructuredText'
     aliases = ['rst', 'rest', 'restructuredtext']
@@ -731,7 +752,7 @@ class RstLexer(RegexLexer):
             (r'^(\s*)(\|)( .+\n(?:\|  .+\n)*)',
              bygroups(Text, Operator, using(this, state='inline'))),
             # Sourcecode directives
-            (r'^( *\.\.)(\s*)((?:source)?code)(::)([ \t]*)([^\n]+)'
+            (r'^( *\.\.)(\s*)((?:source)?code(?:-block)?)(::)([ \t]*)([^\n]+)'
              r'(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\8.*|)\n)+)',
              _handle_sourcecode),
             # A directive
@@ -755,7 +776,7 @@ class RstLexer(RegexLexer):
             (r'^( *)(:.*?:)([ \t]+)(.*?)$',
              bygroups(Text, Name.Class, Text, Name.Function)),
             # Definition list
-            (r'^([^ ].*(?<!::)\n)((?:(?: +.*)\n)+)',
+            (r'^(\S.*(?<!::)\n)((?:(?: +.*)\n)+)',
              bygroups(using(this, state='inline'), using(this, state='inline'))),
             # Code blocks
             (r'(::)(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\3.*|)\n)+)',
@@ -806,7 +827,7 @@ class VimLexer(RegexLexer):
     """
     Lexer for VimL script files.
 
-    *New in Pygments 0.8.*
+    .. versionadded:: 0.8
     """
     name = 'VimL'
     aliases = ['vim']
@@ -815,15 +836,23 @@ class VimLexer(RegexLexer):
     mimetypes = ['text/x-vim']
     flags = re.MULTILINE
 
+    _python = r'py(?:t(?:h(?:o(?:n)?)?)?)?'
+
     tokens = {
         'root': [
+            (r'^([ \t:]*)(' + _python + r')([ \t]*)(<<)([ \t]*)(.*)((?:\n|.)*)(\6)',
+             bygroups(using(this), Keyword, Text, Operator, Text, Text,
+                      using(PythonLexer), Text)), 
+            (r'^([ \t:]*)(' + _python + r')([ \t])(.*)',
+             bygroups(using(this), Keyword, Text, using(PythonLexer))),
+
             (r'^\s*".*', Comment),
 
             (r'[ \t]+', Text),
             # TODO: regexes can have other delims
             (r'/(\\\\|\\/|[^\n/])*/', String.Regex),
             (r'"(\\\\|\\"|[^\n"])*"', String.Double),
-            (r"'(\\\\|\\'|[^\n'])*'", String.Single),
+            (r"'(''|[^\n'])*'", String.Single),
 
             # Who decided that doublequote was a good comment character??
             (r'(?<=\s)"[^\-:.%#=*].*', Comment),
@@ -890,7 +919,7 @@ class GettextLexer(RegexLexer):
     """
     Lexer for Gettext catalog files.
 
-    *New in Pygments 0.9.*
+    .. versionadded:: 0.9
     """
     name = 'Gettext Catalog'
     aliases = ['pot', 'po']
@@ -918,7 +947,7 @@ class SquidConfLexer(RegexLexer):
     """
     Lexer for `squid <http://www.squid-cache.org/>`_ configuration files.
 
-    *New in Pygments 0.9.*
+    .. versionadded:: 0.9
     """
 
     name = 'SquidConf'
@@ -1050,7 +1079,7 @@ class DebianControlLexer(RegexLexer):
     """
     Lexer for Debian ``control`` files and ``apt-cache show <pkg>`` outputs.
 
-    *New in Pygments 0.9.*
+    .. versionadded:: 0.9
     """
     name = 'Debian Control file'
     aliases = ['control', 'debcontrol']
@@ -1120,7 +1149,7 @@ class YamlLexer(ExtendedRegexLexer):
     Lexer for `YAML <http://yaml.org/>`_, a human-friendly data serialization
     language.
 
-    *New in Pygments 0.11.*
+    .. versionadded:: 0.11
     """
 
     name = 'YAML'
@@ -1522,7 +1551,7 @@ class LighttpdConfLexer(RegexLexer):
     """
     Lexer for `Lighttpd <http://lighttpd.net/>`_ configuration files.
 
-    *New in Pygments 0.11.*
+    .. versionadded:: 0.11
     """
     name = 'Lighttpd configuration file'
     aliases = ['lighty', 'lighttpd']
@@ -1550,7 +1579,7 @@ class NginxConfLexer(RegexLexer):
     """
     Lexer for `Nginx <http://nginx.net/>`_ configuration files.
 
-    *New in Pygments 0.11.*
+    .. versionadded:: 0.11
     """
     name = 'Nginx configuration file'
     aliases = ['nginx']
@@ -1596,7 +1625,7 @@ class CMakeLexer(RegexLexer):
     """
     Lexer for `CMake <http://cmake.org/Wiki/CMake>`_ files.
 
-    *New in Pygments 1.2.*
+    .. versionadded:: 1.2
     """
     name = 'CMake'
     aliases = ['cmake']
@@ -1640,6 +1669,7 @@ class CMakeLexer(RegexLexer):
             (r'\(', Punctuation, '#push'),
             (r'\)', Punctuation, '#pop'),
             (r'(\${)(.+?)(})', bygroups(Operator, Name.Variable, Operator)),
+            (r'(\$<)(.+?)(>)', bygroups(Operator, Name.Variable, Operator)),
             (r'(?s)".*?"', String.Double),
             (r'\\\S+', String),
             (r'[^\)$"# \t\n]+', String),
@@ -1656,16 +1686,22 @@ class CMakeLexer(RegexLexer):
         ],
         'ws': [
             (r'[ \t]+', Text),
-            (r'#.+\n', Comment),
+            (r'#.*\n', Comment),
         ]
     }
 
+    def analyse_text(text):
+        exp = r'^ *CMAKE_MINIMUM_REQUIRED *\( *VERSION *\d(\.\d)* *( FATAL_ERROR)? *\) *$'
+        if re.search(exp, text, flags=re.MULTILINE | re.IGNORECASE):
+            return 0.8
+        return 0.0
+
 
 class HttpLexer(RegexLexer):
     """
     Lexer for HTTP sessions.
 
-    *New in Pygments 1.5.*
+    .. versionadded:: 1.5
     """
 
     name = 'HTTP'
@@ -1734,7 +1770,7 @@ class PyPyLogLexer(RegexLexer):
     """
     Lexer for PyPy log files.
 
-    *New in Pygments 1.5.*
+    .. versionadded:: 1.5
     """
     name = "PyPy Log"
     aliases = ["pypylog", "pypy"]
@@ -1806,7 +1842,7 @@ class HxmlLexer(RegexLexer):
     """
     Lexer for `haXe build <http://haxe.org/doc/compiler>`_ files.
 
-    *New in Pygments 1.6.*
+    .. versionadded:: 1.6
     """
     name = 'Hxml'
     aliases = ['haxeml', 'hxml']
@@ -1849,7 +1885,7 @@ class EbnfLexer(RegexLexer):
     <http://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_Form>`_
     grammars.
 
-    *New in Pygments 1.7.*
+    .. versionadded:: 2.0
     """
 
     name = 'EBNF'
@@ -1891,3 +1927,129 @@ class EbnfLexer(RegexLexer):
             (r'([a-zA-Z][a-zA-Z0-9 \-]*)', Keyword),
           ],
     }
+
+class TodotxtLexer(RegexLexer):
+    """
+    Lexer for `Todo.txt <http://todotxt.com/>`_ todo list format.
+
+    .. versionadded:: 2.0
+    """
+
+    name = 'Todotxt'
+    aliases = ['todotxt']
+    # *.todotxt is not a standard extension for Todo.txt files; including it
+    # makes testing easier, and also makes autodetecting file type easier.
+    filenames = ['todo.txt', '*.todotxt']
+    mimetypes = ['text/x-todo']
+
+    ## Aliases mapping standard token types of Todo.txt format concepts
+    CompleteTaskText = Operator # Chosen to de-emphasize complete tasks
+    IncompleteTaskText = Text # Incomplete tasks should look like plain text
+
+    # Priority should have most emphasis to indicate importance of tasks
+    Priority = Generic.Heading
+    # Dates should have next most emphasis because time is important
+    Date = Generic.Subheading
+
+    # Project and context should have equal weight, and be in different colors
+    Project = Generic.Error
+    Context = String
+
+    # If tag functionality is added, it should have the same weight as Project
+    # and Context, and a different color. Generic.Traceback would work well.
+
+    # Regex patterns for building up rules; dates, priorities, projects, and
+    # contexts are all atomic
+    # TODO: Make date regex more ISO 8601 compliant
+    date_regex = r'\d{4,}-\d{2}-\d{2}'
+    priority_regex = r'\([A-Z]\)'
+    project_regex = r'\+\S+'
+    context_regex = r'@\S+'
+
+    # Compound regex expressions
+    complete_one_date_regex = r'(x )(' + date_regex + r')'
+    complete_two_date_regex = (complete_one_date_regex + r'( )(' +
+                                date_regex + r')')
+    priority_date_regex = r'(' + priority_regex + r')( )(' + date_regex + r')'
+
+    tokens = {
+        # Should parse starting at beginning of line; each line is a task
+        'root': [
+            ## Complete task entry points: two total:
+            # 1. Complete task with two dates
+            (complete_two_date_regex, bygroups(CompleteTaskText, Date,
+                                               CompleteTaskText, Date),
+             'complete'),
+            # 2. Complete task with one date
+            (complete_one_date_regex, bygroups(CompleteTaskText, Date),
+             'complete'),
+
+            ## Incomplete task entry points: six total:
+            # 1. Priority plus date
+            (priority_date_regex, bygroups(Priority, IncompleteTaskText, Date),
+             'incomplete'),
+            # 2. Priority only
+            (priority_regex, Priority, 'incomplete'),
+            # 3. Leading date
+            (date_regex, Date, 'incomplete'),
+            # 4. Leading context
+            (context_regex, Context, 'incomplete'),
+            # 5. Leading project
+            (project_regex, Project, 'incomplete'),
+            # 6. Non-whitespace catch-all
+            ('\S+', IncompleteTaskText, 'incomplete'),
+        ],
+
+        # Parse a complete task
+        'complete': [
+            # Newline indicates end of task, should return to root
+            (r'\s*\n', CompleteTaskText, '#pop'),
+            # Tokenize contexts and projects
+            (context_regex, Context),
+            (project_regex, Project),
+            # Tokenize non-whitespace text
+            ('\S+', CompleteTaskText),
+            # Tokenize whitespace not containing a newline
+            ('\s+', CompleteTaskText),
+        ],
+
+        # Parse an incomplete task
+        'incomplete': [
+            # Newline indicates end of task, should return to root
+            (r'\s*\n', IncompleteTaskText, '#pop'),
+            # Tokenize contexts and projects
+            (context_regex, Context),
+            (project_regex, Project),
+            # Tokenize non-whitespace text
+            ('\S+', IncompleteTaskText),
+            # Tokenize whitespace not containing a newline
+            ('\s+', IncompleteTaskText),
+        ],
+    }
+
+
+class DockerLexer(RegexLexer):
+    """
+    Lexer for `Docker <http://docker.io>`_ configuration files.
+
+    .. versionadded:: 2.0
+    """
+    name = 'Docker'
+    aliases = ['docker', 'dockerfile']
+    filenames = ['Dockerfile', '*.docker']
+    mimetypes = ['text/x-dockerfile-config']
+
+    _keywords = (r'(?:FROM|MAINTAINER|RUN|CMD|EXPOSE|ENV|ADD|ENTRYPOINT|'
+                 r'VOLUME|WORKDIR)')
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    tokens = {
+        'root': [
+            (r'^(ONBUILD)(\s+)(%s)\b' % (_keywords,),
+             bygroups(Name.Keyword, Whitespace, Keyword)),
+            (_keywords + r'\b', Keyword),
+            (r'#.*', Comment),
+            (r'.+', using(BashLexer)),
+        ],
+    }