gitlab-pygments.rb 0.3.2

data/test/test_pygments.rb

@@ -0,0 +1,276 @@
+#coding: utf-8
+
+require 'test/unit'
+require File.join(File.dirname(__FILE__), '..', '/lib/pygments.rb')
+ENV['mentos-test'] = "yes"
+
+P = Pygments
+
+class PygmentsHighlightTest < Test::Unit::TestCase
+  RUBY_CODE = "#!/usr/bin/ruby\nputs 'foo'"
+  RUBY_CODE_TRAILING_NEWLINE = "#!/usr/bin/ruby\nputs 'foo'\n"
+  REDIS_CODE = File.read(File.join(File.dirname(__FILE__), '..', '/test/test_data.c'))
+
+  def test_highlight_defaults_to_html
+    code = P.highlight(RUBY_CODE)
+    assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
+    assert_equal '<div class', code[0..9]
+  end
+
+  def test_highlight_works_with_larger_files
+    code = P.highlight(REDIS_CODE)
+    assert_match 'used_memory_peak_human', code
+    assert_equal 454107, code.bytesize.to_i
+  end
+
+  def test_returns_nil_on_timeout
+    large_code = REDIS_CODE * 300
+    code = P.highlight(large_code) # a 30 mb highlight request will timeout
+    assert_equal nil, code
+  end
+
+  def test_highlight_works_with_null_bytes
+    code = P.highlight("\0hello", :lexer => 'rb')
+    assert_match "hello", code
+  end
+
+  def test_highlight_works_on_utf8
+    code = P.highlight('# ø', :lexer => 'rb', :options => {:encoding => 'utf-8'})
+    assert_match "# ø", code
+  end
+
+  def test_highlight_works_on_utf8_automatically
+    code = P.highlight('# ø', :lexer => 'rb')
+    assert_match "# ø", code
+  end
+
+  def test_highlight_works_on_utf8_all_chars_automatically
+    code = P.highlight('def foo: # ø', :lexer => 'py')
+
+    assert_equal '<div class="highlight"><pre><span clas', code[0,38]
+  end
+
+  def test_highlight_works_with_multiple_utf8
+    code = P.highlight('# ø ø ø', :lexer => 'rb', :options => {:encoding => 'utf-8'})
+    assert_match "# ø ø ø", code
+  end
+
+  def test_highlight_works_with_multiple_utf8_and_trailing_newline
+    code = P.highlight("#!/usr/bin/ruby\nputs 'ø..ø'\n", :lexer => 'rb')
+    assert_match "ø..ø", code
+  end
+
+  def test_highlight_formatter_bbcode
+    code = P.highlight(RUBY_CODE, :formatter => 'bbcode')
+    assert_match 'color=#408080][i]#!/usr/bin/ruby[/i]', code
+  end
+
+  def test_highlight_formatter_terminal
+    code = P.highlight(RUBY_CODE, :formatter => 'terminal')
+    assert_match '39;49;00m', code
+  end
+
+  def test_highlight_options
+    code = P.highlight(RUBY_CODE, :options => {:full => true, :title => 'test'})
+    assert_match '<title>test</title>', code
+  end
+
+  def test_highlight_works_with_single_character_input
+    code = P.highlight("a")
+    assert_match 'a</span>', code
+  end
+
+  def test_highlight_works_with_trailing_newline
+    code = P.highlight(RUBY_CODE_TRAILING_NEWLINE)
+    assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
+  end
+
+  def test_highlight_works_with_multiple_newlines
+    code = P.highlight(RUBY_CODE_TRAILING_NEWLINE + "derp\n\n")
+    assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
+  end
+
+  def test_highlight_works_with_trailing_cr
+    code = P.highlight(RUBY_CODE_TRAILING_NEWLINE + "\r")
+    assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
+  end
+
+  def test_highlight_still_works_with_invalid_code
+    code = P.highlight("importr python;    wat?", :lexer => 'py')
+    assert_match ">importr</span>", code
+  end
+end
+
+# Philosophically, I'm not the biggest fan of testing private
+# methods, but given the relative delicacy of validity checking
+# over the pipe I think it's necessary and informative.
+class PygmentsValidityTest < Test::Unit::TestCase
+  def test_add_ids_with_padding
+    res = P.send(:add_ids, "herp derp baz boo foo", "ABCDEFGH")
+    assert_equal "ABCDEFGH  herp derp baz boo foo  ABCDEFGH", res
+  end
+
+  def test_add_ids_on_empty_string
+    res = P.send(:add_ids, "", "ABCDEFGH")
+    assert_equal "ABCDEFGH    ABCDEFGH", res
+  end
+
+  def test_add_ids_with_unicode_data
+    res = P.send(:add_ids, "# ø ø ø", "ABCDEFGH")
+    assert_equal "ABCDEFGH  # ø ø ø  ABCDEFGH", res
+  end
+
+  def test_add_ids_with_starting_slashes
+    res = P.send(:add_ids, '\\# ø ø ø..//', "ABCDEFGH")
+    assert_equal "ABCDEFGH  \\# ø ø ø..//  ABCDEFGH", res
+  end
+
+  def test_get_fixed_bits_from_header
+    bits = P.send(:get_fixed_bits_from_header, '{"herp": "derp"}')
+    assert_equal "00000000000000000000000000010000", bits
+  end
+
+  def test_get_fixed_bits_from_header_works_with_large_headers
+    bits = P.send(:get_fixed_bits_from_header, '{"herp": "derp"}' * 10000)
+    assert_equal "00000000000000100111000100000000", bits
+  end
+
+  def test_size_check
+    size = "00000000000000000000000000100110"
+    res = P.send(:size_check, size)
+    assert_equal res, true
+  end
+
+  def test_size_check_bad
+    size = "some random thing"
+    res = P.send(:size_check, size)
+    assert_equal res, false
+  end
+end
+
+class PygmentsLexerTest < Test::Unit::TestCase
+  RUBY_CODE = "#!/usr/bin/ruby\nputs 'foo'"
+
+  def test_lexer_by_mimetype
+    assert_equal 'rb', P.lexer_name_for(:mimetype => 'text/x-ruby')
+    assert_equal 'json', P.lexer_name_for(:mimetype => 'application/json')
+  end
+
+  def test_lexer_by_filename
+    assert_equal 'rb', P.lexer_name_for(:filename => 'test.rb')
+    assert_equal 'scala', P.lexer_name_for(:filename => 'test.scala')
+  end
+
+  def test_lexer_by_name
+    assert_equal 'rb', P.lexer_name_for(:lexer => 'ruby')
+    assert_equal 'python', P.lexer_name_for(:lexer => 'python')
+    assert_equal 'c', P.lexer_name_for(:lexer => 'c')
+  end
+
+  def test_lexer_by_filename_and_content
+    assert_equal 'rb', P.lexer_name_for(RUBY_CODE, :filename => 'test.rb')
+  end
+
+  def test_lexer_by_content
+    assert_equal 'rb', P.lexer_name_for(RUBY_CODE)
+  end
+
+  def test_lexer_by_nothing
+    assert_raise MentosError do
+      P.lexer_name_for(:invalid => true)
+    end
+  end
+end
+
+class PygmentsLexerClassTest < Test::Unit::TestCase
+  def test_find
+    assert_equal 'Ruby', P::Lexer['Ruby'].name
+    assert_equal 'Ruby', P::Lexer['ruby'].name
+    assert_equal 'Ruby', P::Lexer['rb'].name
+    assert_equal 'Ruby', P::Lexer['rake'].name
+    assert_equal 'Ruby', P::Lexer['gemspec'].name
+  end
+
+  def test_find_by_name
+    assert_equal P::Lexer['Ruby'], P::Lexer.find_by_name('Ruby')
+    assert_equal P::Lexer['C'], P::Lexer.find_by_name('C')
+  end
+
+  def test_find_by_alias
+    assert_equal P::Lexer['Ruby'], P::Lexer.find_by_alias('rb')
+    assert_equal P::Lexer['Ruby'], P::Lexer.find_by_alias('ruby')
+    assert_equal P::Lexer['Scala'], P::Lexer.find_by_alias('scala')
+    assert_equal P::Lexer['Go'], P::Lexer.find_by_alias('go')
+  end
+
+  def test_find_lexer_by_extname
+    assert_equal P::Lexer['Ruby'], P::Lexer.find_by_extname('.rb')
+    assert_equal P::Lexer['PHP'], P::Lexer.find_by_extname('.php4')
+    assert_equal P::Lexer['PHP'], P::Lexer.find_by_extname('.php5')
+    assert_equal P::Lexer['Groff'], P::Lexer.find_by_extname('.1')
+    assert_equal P::Lexer['Groff'], P::Lexer.find_by_extname('.3')
+    assert_equal P::Lexer['C'], P::Lexer.find_by_extname('.c')
+    assert_equal P::Lexer['C'], P::Lexer.find_by_extname('.h')
+    assert_equal P::Lexer['Python'], P::Lexer.find_by_extname('.py')
+    assert_equal P::Lexer['Java'], P::Lexer.find_by_extname('.java')
+  end
+
+ def test_find_lexer_by_mimetype
+    assert_equal P::Lexer['Ruby'], P::Lexer.find_by_mimetype('text/x-ruby')
+    assert_equal P::Lexer['JSON'], P::Lexer.find_by_mimetype('application/json')
+    assert_equal P::Lexer['Python'], P::Lexer.find_by_mimetype('text/x-python')
+  end
+end
+
+
+class PygmentsCssTest < Test::Unit::TestCase
+  include Pygments
+
+  def test_css
+    assert_match /^\.err \{/, P.css
+  end
+
+  def test_css_prefix
+    assert_match /^\.highlight \.err \{/, P.css('.highlight')
+  end
+
+  def test_css_options
+    assert_match /^\.codeerr \{/, P.css(:classprefix => 'code')
+  end
+
+  def test_css_prefix_and_options
+    assert_match /^\.mycode \.codeerr \{/, P.css('.mycode', :classprefix => 'code')
+  end
+
+  def test_css_default
+    assert_match '.c { color: #408080; font-style: italic }', P.css
+  end
+
+  def test_css_colorful
+    assert_match '.c { color: #888888 }', P.css(:style => 'colorful')
+  end
+end
+
+class PygmentsConfigTest < Test::Unit::TestCase
+  def test_styles
+    assert P.styles.include?('colorful')
+  end
+
+  def test_filters
+    assert P.filters.include?('codetagify')
+  end
+
+  def test_lexers
+    list = P.lexers
+    assert list.has_key?('Ruby')
+    assert list['Ruby'][:aliases].include?('duby')
+  end
+
+  def test_formatters
+    list = P.formatters
+    assert list.has_key?('Html')
+    assert list['Html'][:aliases].include?('html')
+  end
+end
+
+

data/vendor/custom_formatters/gitlab.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.formatters.gitlab
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    GitLab specific formatter for HTML output.
+    Based on the standard HTML formatter.
+
+    :copyright: Copyright 2012 by the GitLab team (http://www.gitlab.org).
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import sys
+import StringIO
+
+from pygments.formatter import Formatter
+from pygments.token import Token, Text, STANDARD_TYPES
+from pygments.util import get_bool_opt, get_int_opt
+
+
+__all__ = ['GitlabFormatter']
+
+
+_escape_html_table = {
+    ord('&'): u'&',
+    ord('<'): u'&lt;',
+    ord('>'): u'&gt;',
+    ord('"'): u'&quot;',
+    ord("'"): u'&#39;',
+}
+
+def escape_html(text, table=_escape_html_table):
+    """Escape &, <, > as well as single and double quotes for HTML."""
+    return text.translate(table)
+
+
+def _get_ttype_class(ttype):
+    fname = STANDARD_TYPES.get(ttype)
+    if fname:
+        return fname
+    aname = ''
+    while fname is None:
+        aname = '-' + ttype[-1] + aname
+        ttype = ttype.parent
+        fname = STANDARD_TYPES.get(ttype)
+    return fname + aname
+
+
+class GitlabFormatter(Formatter):
+    r"""
+    GitLab specific formatter for HTML output.
+
+    Additional options accepted:
+
+    `show_line_numbers`
+        Determines whether the line number column should be shown (default:
+        ``True``).
+
+    `first_line_number`
+        The line number for the first line (default: ``1``).
+    """
+
+    name = 'GitLab'
+    aliases = ['gitlab']
+    filenames = []
+
+    def __init__(self, **options):
+        Formatter.__init__(self, **options)
+        self.show_line_numbers = get_bool_opt(options, 'show_line_numbers', True)
+        self.first_line_number = abs(get_int_opt(options, 'first_line_number', 1))
+
+    def _wrap_table(self, inner):
+        """
+        Wrap the whole thing into a table and add line numbers
+        """
+        dummyoutfile = StringIO.StringIO()
+        lncount = 0
+        for line in inner:
+            lncount += 1
+            dummyoutfile.write(line)
+
+        sln = self.show_line_numbers
+        if sln:
+          fl = self.first_line_number
+          mw = len(str(lncount + fl - 1))
+
+          lines = []
+          for i in range(fl, fl+lncount):
+              lines.append('<a id="L%d" href="#L%d" rel="#L%d">' % (i, i, i) +
+                            '<i class="icon-link"></i> %*d' % (mw, i) +
+                            '</a>')
+          ls = '\n'.join(lines)
+
+        yield '<table class="lines"><tr>'
+        if sln:
+          yield '<td><pre class="line_numbers">' + ls + '</pre></td>'
+        yield '<td><div class="highlight"><pre>'
+        yield dummyoutfile.getvalue()
+        yield '</pre></div></td></tr></table>'
+
+    def _wrap_code_lines(self, inner):
+        """
+        Wrap each line in a <div class="line">
+        """
+        # subtract 1 since we have to increment i *before* yielding
+        i = self.first_line_number - 1
+
+        for line in inner:
+            i += 1
+            yield '<div id="LC%d" class="line">%s</div>' % (i, line)
+
+    def _format_code_lines(self, tokensource):
+        """
+        Just format the tokens, without any wrapping tags.
+        Yield individual lines.
+        """
+        # for <span style=""> lookup only
+        escape_table = _escape_html_table
+
+        lspan = ''
+        line = ''
+        for ttype, value in tokensource:
+            cls = _get_ttype_class(ttype)
+            cspan = cls and '<span class="%s">' % cls or ''
+
+            parts = escape_html(value).split('\n')
+
+            # for all but the last line
+            for part in parts[:-1]:
+                if line:
+                    if lspan != cspan:
+                        line += (lspan and '</span>') + cspan + part + \
+                                (cspan and '</span>')
+                    else: # both are the same
+                        line += part + (lspan and '</span>')
+                    yield line
+                    line = ''
+                elif part:
+                    yield cspan + part + (cspan and '</span>')
+                else:
+                    yield '<br/>'
+            # for the last line
+            if line and parts[-1]:
+                if lspan != cspan:
+                    line += (lspan and '</span>') + cspan + parts[-1]
+                    lspan = cspan
+                else:
+                    line += parts[-1]
+            elif parts[-1]:
+                line = cspan + parts[-1]
+                lspan = cspan
+            # else we neither have to open a new span nor set lspan
+
+        if line:
+            yield line + (lspan and '</span>')
+
+    def format_unencoded(self, tokensource, outfile):
+        """
+        The formatting process uses several nested generators; which of
+        them are used is determined by the user's options.
+
+        Each generator should take at least one argument, ``inner``,
+        and wrap the pieces of text generated by this.
+        """
+        source = self._format_code_lines(tokensource)
+        source = self._wrap_code_lines(source)
+        source = self._wrap_table(source)
+
+        for piece in source:
+            outfile.write(piece)

data/vendor/custom_lexers/github.py

@@ -0,0 +1,362 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.github
+    ~~~~~~~~~~~~~~~~~~~
+
+    Custom lexers for GitHub.com
+
+    :copyright: Copyright 2012 by GitHub, Inc
+    :license: BSD, see LICENSE for details.
+"""
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, DelegatingLexer
+from pygments.token import Text, Name, Number, String, Comment, Punctuation, \
+     Other, Keyword, Operator, Literal
+
+__all__ = ['Dasm16Lexer', 'PuppetLexer', 'AugeasLexer']
+
+class Dasm16Lexer(RegexLexer):
+    """
+    Simple lexer for DCPU-16 Assembly
+
+    Check http://0x10c.com/doc/dcpu-16.txt
+    """
+    name = 'dasm16'
+    aliases = ['DASM16']
+    filenames = ['*.dasm16', '*.dasm']
+    mimetypes = ['text/x-dasm16']
+
+    INSTRUCTIONS = [
+        'SET',
+        'ADD', 'SUB',
+        'MUL', 'MLI',
+        'DIV', 'DVI',
+        'MOD', 'MDI',
+        'AND', 'BOR', 'XOR',
+        'SHR', 'ASR', 'SHL',
+        'IFB', 'IFC', 'IFE', 'IFN', 'IFG', 'IFA', 'IFL', 'IFU',
+        'ADX', 'SBX',
+        'STI', 'STD',
+        'JSR',
+        'INT', 'IAG', 'IAS', 'RFI', 'IAQ', 'HWN', 'HWQ', 'HWI',
+    ]
+
+    REGISTERS = [
+        'A', 'B', 'C',
+        'X', 'Y', 'Z',
+        'I', 'J',
+        'SP', 'PC', 'EX',
+        'POP', 'PEEK', 'PUSH'
+    ]
+
+    # Regexes yo
+    char = r'[a-zA-Z$._0-9@]'
+    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
+    number = r'[+-]?(?:0[xX][a-zA-Z0-9]+|\d+)'
+    binary_number = r'0b[01_]+'
+    instruction = r'(?i)(' + '|'.join(INSTRUCTIONS) + ')'
+    single_char = r"'\\?" + char + "'"
+    string = r'"(\\"|[^"])*"'
+
+    def guess_identifier(lexer, match):
+        ident = match.group(0)
+        klass = Name.Variable if ident.upper() in lexer.REGISTERS else Name.Label
+        yield match.start(), klass, ident
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            (':' + identifier, Name.Label),
+            (identifier + ':', Name.Label),
+            (instruction, Name.Function, 'instruction-args'),
+            (r'\.' + identifier, Name.Function, 'data-args'),
+            (r'[\r\n]+', Text)
+        ],
+
+        'numeric' : [
+            (binary_number, Number.Integer),
+            (number, Number.Integer),
+            (single_char, String),
+        ],
+
+        'arg' : [
+            (identifier, guess_identifier),
+            include('numeric')
+        ],
+
+        'deref' : [
+            (r'\+', Punctuation),
+            (r'\]', Punctuation, '#pop'),
+            include('arg'),
+            include('whitespace')
+        ],
+
+        'instruction-line' : [
+            (r'[\r\n]+', Text, '#pop'),
+            (r';.*?$', Comment, '#pop'),
+            include('whitespace')
+        ],
+
+        'instruction-args': [
+            (r',', Punctuation),
+            (r'\[', Punctuation, 'deref'),
+            include('arg'),
+            include('instruction-line')
+        ],
+
+        'data-args' : [
+            (r',', Punctuation),
+            include('numeric'),
+            (string, String),
+            include('instruction-line')
+        ],
+
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r';.*?\n', Comment)
+        ],
+    }
+
+class PuppetLexer(RegexLexer):
+    name = 'Puppet'
+    aliases = ['puppet']
+    filenames = ['*.pp']
+
+    tokens = {
+        'root': [
+            include('puppet'),
+        ],
+        'puppet': [
+            include('comments'),
+            (r'(class)(\s*)(\{)', bygroups(Name.Class, Text, Punctuation), ('type', 'namevar')),
+            (r'(class|define)', Keyword.Declaration, ('block','class_name')),
+            (r'node', Keyword.Declaration, ('block', 'node_name')),
+            (r'elsif', Keyword.Reserved, ('block', 'conditional')),
+            (r'if', Keyword.Reserved, ('block', 'conditional')),
+            (r'unless', Keyword.Reserved, ('block', 'conditional')),
+            (r'(else)(\s*)(\{)', bygroups(Keyword.Reserved, Text, Punctuation), 'block'),
+            (r'case', Keyword.Reserved, ('case', 'conditional')),
+            (r'(::)?([A-Z][\w:]+)+(\s*)(<{1,2}\|)', bygroups(Name.Class, Name.Class, Text, Punctuation), 'spaceinvader'),
+            (r'(::)?([A-Z][\w:]+)+(\s*)(\{)', bygroups(Name.Class, Name.Class, Text, Punctuation), 'type'),
+            (r'(::)?([A-Z][\w:]+)+(\s*)(\[)', bygroups(Name.Class, Name.Class, Text, Punctuation), ('type', 'override_name')),
+            (r'(@{0,2}[\w:]+)(\s*)(\{)(\s*)', bygroups(Name.Class, Text, Punctuation, Text), ('type', 'namevar')),
+            (r'\$(::)?(\w+::)*\w+', Name.Variable, 'var_assign'),
+            (r'(include|require)', Keyword.Namespace, 'include'),
+            (r'import', Keyword.Namespace, 'import'),
+            (r'(\w+)(\()', bygroups(Name.Function, Punctuation), 'function'),
+            (r'\s', Text),
+        ],
+        'block': [
+            include('puppet'),
+            (r'\}', Text, '#pop'),
+        ],
+        'override_name': [
+            include('strings'),
+            include('variables'),
+            (r'\]', Punctuation),
+            (r'\s', Text),
+            (r'\{', Punctuation, '#pop'),
+        ],
+        'node_name': [
+            (r'inherits', Keyword.Declaration),
+            (r'[\w\.]+', String),
+            include('strings'),
+            include('variables'),
+            (r',', Punctuation),
+            (r'\s', Text),
+            (r'\{', Punctuation, '#pop'),
+        ],
+        'class_name': [
+            (r'inherits', Keyword.Declaration),
+            (r'[\w:]+', Name.Class),
+            (r'\s', Text),
+            (r'\{', Punctuation, '#pop'),
+            (r'\(', Punctuation, 'paramlist'),
+        ],
+        'include': [
+            (r'\n', Text, '#pop'),
+            (r'[\w:-]+', Name.Class),
+            include('value'),
+            (r'\s', Text),
+        ],
+        'import': [
+            (r'\n', Text, '#pop'),
+            (r'[\/\w\.]+', String),
+            include('value'),
+            (r'\s', Text),
+        ],
+        'case': [
+            (r'(default)(:)(\s*)(\{)', bygroups(Keyword.Reserved, Punctuation, Text, Punctuation), 'block'),
+            include('case_values'),
+            (r'(:)(\s*)(\{)', bygroups(Punctuation, Text, Punctuation), 'block'),
+            (r'\s', Text),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'case_values': [
+            include('value'),
+            (r',', Punctuation),
+        ],
+        'comments': [
+            (r'\s*#.*\n', Comment.Singleline),
+        ],
+        'strings': [
+            (r"'.*?'", String.Single),
+            (r'\w+', String.Symbol),
+            (r'"', String.Double, 'dblstring'),
+            (r'\/.+?\/', String.Regex),
+        ],
+        'dblstring': [
+            (r'\$\{.+?\}', String.Interpol),
+            (r'(?:\\(?:[bdefnrstv\'"\$\\/]|[0-7][0-7]?[0-7]?|\^[a-zA-Z]))', String.Escape),
+            (r'[^"\\\$]+', String.Double),
+            (r'\$', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'variables': [
+            (r'\$(::)?(\w+::)*\w+', Name.Variable),
+        ],
+        'var_assign': [
+            (r'\[', Punctuation, ('#pop', 'array')),
+            (r'\{', Punctuation, ('#pop', 'hash')),
+            (r'(\s*)(=)(\s*)', bygroups(Text, Operator, Text)),
+            (r'(\(|\))', Punctuation),
+            include('operators'),
+            include('value'),
+            (r'\s', Text, '#pop'),
+        ],
+        'booleans': [
+            (r'(true|false)', Literal),
+        ],
+        'operators': [
+            (r'(\s*)(==|=~|\*|-|\+|<<|>>|!=|!~|!|>=|<=|<|>|and|or|in)(\s*)', bygroups(Text, Operator, Text)),
+        ],
+        'conditional': [
+            include('operators'),
+            include('strings'),
+            include('variables'),
+            (r'\[', Punctuation, 'array'),
+            (r'\(', Punctuation, 'conditional'),
+            (r'\{', Punctuation, '#pop'),
+            (r'\)', Punctuation, '#pop'),
+            (r'\s', Text),
+        ],
+        'spaceinvader': [
+            include('operators'),
+            include('strings'),
+            include('variables'),
+            (r'\[', Punctuation, 'array'),
+            (r'\(', Punctuation, 'conditional'),
+            (r'\s', Text),
+            (r'\|>{1,2}', Punctuation, '#pop'),
+        ],
+        'namevar': [
+            include('value'),
+            (r'\[', Punctuation, 'array'),
+            (r'\s', Text),
+            (r':', Punctuation, '#pop'),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'function': [
+            (r'\[', Punctuation, 'array'),
+            include('value'),
+            (r',', Punctuation),
+            (r'\s', Text),
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'paramlist': [
+            include('value'),
+            (r'=', Punctuation),
+            (r',', Punctuation),
+            (r'\s', Text),
+            (r'\[', Punctuation, 'array'),
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'type': [
+            (r'(\w+)(\s*)(=>)(\s*)', bygroups(Name.Tag, Text, Punctuation, Text), 'param_value'),
+            (r'\}', Punctuation, '#pop'),
+            (r'\s', Text),
+            include('comments'),
+            (r'', Text, 'namevar'),
+        ],
+        'value': [
+            (r'[\d\.]', Number),
+            (r'([A-Z][\w:]+)+(\[)', bygroups(Name.Class, Punctuation), 'array'),
+            (r'(\w+)(\()', bygroups(Name.Function, Punctuation), 'function'),
+            include('strings'),
+            include('variables'),
+            include('comments'),
+            include('booleans'),
+            (r'(\s*)(\?)(\s*)(\{)', bygroups(Text, Punctuation, Text, Punctuation), 'selector'),
+            (r'\{', Punctuation, 'hash'),
+        ],
+        'selector': [
+            (r'default', Keyword.Reserved),
+            include('value'),
+            (r'=>', Punctuation),
+            (r',', Punctuation),
+            (r'\s', Text),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'param_value': [
+            include('value'),
+            (r'\[', Punctuation, 'array'),
+            (r',', Punctuation, '#pop'),
+            (r';', Punctuation, '#pop'),
+            (r'\s', Text, '#pop'),
+            (r'', Text, '#pop'),
+        ],
+        'array': [
+            include('value'),
+            (r'\[', Punctuation, 'array'),
+            (r',', Punctuation),
+            (r'\s', Text),
+            (r'\]', Punctuation, '#pop'),
+        ],
+        'hash': [
+            include('value'),
+            (r'\s', Text),
+            (r'=>', Punctuation),
+            (r',', Punctuation),
+            (r'\}', Punctuation, '#pop'),
+        ],
+    }
+
+class AugeasLexer(RegexLexer):
+    name = 'Augeas'
+    aliases = ['augeas']
+    filenames = ['*.aug']
+
+    tokens = {
+        'root': [
+            (r'(module)(\s*)([^\s=]+)', bygroups(Keyword.Namespace, Text, Name.Namespace)),
+            (r'(let)(\s*)([^\s=]+)', bygroups(Keyword.Declaration, Text, Name.Variable)),
+            (r'(del|store|value|counter|seq|key|label|autoload|incl|excl|transform|test|get|put)(\s+)', bygroups(Name.Builtin, Text)),
+            (r'(\()([^\:]+)(\:)(unit|string|regexp|lens|tree|filter)(\))', bygroups(Punctuation, Name.Variable, Punctuation, Keyword.Type, Punctuation)),
+            (r'\(\*', Comment.Multiline, 'comment'),
+            (r'[\+=\|\.\*\;\?-]', Operator),
+            (r'[\[\]\(\)\{\}]', Operator),
+            (r'"', String.Double, 'string'),
+            (r'\/', String.Regex, 'regex'),
+            (r'([A-Z]\w*)(\.)(\w+)', bygroups(Name.Namespace, Punctuation, Name.Variable)),
+            (r'.', Name.Variable),
+            (r'\s', Text),
+        ],
+        'string': [
+            (r'\\.', String.Escape),
+            (r'[^"]', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'regex': [
+            (r'\\.', String.Escape),
+            (r'[^\/]', String.Regex),
+            (r'\/', String.Regex, '#pop'),
+        ],
+        'comment': [
+            (r'[^*\)]', Comment.Multiline),
+            (r'\(\*', Comment.Multiline, '#push'),
+            (r'\*\)', Comment.Multiline, '#pop'),
+            (r'[\*\)]', Comment.Multiline)
+        ],
+    }