mortar-pygments.rb 0.5.5

data/vendor/pygments-main/tests/test_examplefiles.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments tests with example files
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import pprint
+import difflib
+import cPickle as pickle
+
+from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
+from pygments.token import Error
+from pygments.util import ClassNotFound, b
+
+STORE_OUTPUT = False
+
+# generate methods
+def test_example_files():
+    testdir = os.path.dirname(__file__)
+    outdir = os.path.join(testdir, 'examplefiles', 'output')
+    if STORE_OUTPUT and not os.path.isdir(outdir):
+        os.makedirs(outdir)
+    for fn in os.listdir(os.path.join(testdir, 'examplefiles')):
+        if fn.startswith('.') or fn.endswith('#'):
+            continue
+
+        absfn = os.path.join(testdir, 'examplefiles', fn)
+        if not os.path.isfile(absfn):
+            continue
+        outfn = os.path.join(outdir, fn)
+
+        try:
+            lx = get_lexer_for_filename(absfn)
+        except ClassNotFound:
+            if "_" not in fn:
+                raise AssertionError('file %r has no registered extension, '
+                                     'nor is of the form <lexer>_filename '
+                                     'for overriding, thus no lexer found.'
+                                    % fn)
+            try:
+                name, rest = fn.split("_", 1)
+                lx = get_lexer_by_name(name)
+            except ClassNotFound:
+                raise AssertionError('no lexer found for file %r' % fn)
+        yield check_lexer, lx, absfn, outfn
+
+def check_lexer(lx, absfn, outfn):
+    fp = open(absfn, 'rb')
+    try:
+        text = fp.read()
+    finally:
+        fp.close()
+    text = text.replace(b('\r\n'), b('\n'))
+    text = text.strip(b('\n')) + b('\n')
+    try:
+        text = text.decode('utf-8')
+        if text.startswith(u'\ufeff'):
+            text = text[len(u'\ufeff'):]
+    except UnicodeError:
+        text = text.decode('latin1')
+    ntext = []
+    tokens = []
+    for type, val in lx.get_tokens(text):
+        ntext.append(val)
+        assert type != Error, \
+            'lexer %s generated error token for %s: %r at position %d' % \
+            (lx, absfn, val, len(u''.join(ntext)))
+        tokens.append((type, val))
+    if u''.join(ntext) != text:
+        print '\n'.join(difflib.unified_diff(u''.join(ntext).splitlines(),
+                                             text.splitlines()))
+        raise AssertionError('round trip failed for ' + absfn)
+
+    # check output against previous run if enabled
+    if STORE_OUTPUT:
+        # no previous output -- store it
+        if not os.path.isfile(outfn):
+            fp = open(outfn, 'wb')
+            try:
+                pickle.dump(tokens, fp)
+            finally:
+                fp.close()
+            return
+        # otherwise load it and compare
+        fp = open(outfn, 'rb')
+        try:
+            stored_tokens = pickle.load(fp)
+        finally:
+            fp.close()
+        if stored_tokens != tokens:
+            f1 = pprint.pformat(stored_tokens)
+            f2 = pprint.pformat(tokens)
+            print '\n'.join(difflib.unified_diff(f1.splitlines(),
+                                                 f2.splitlines()))
+            assert False, absfn

data/vendor/pygments-main/tests/test_html_formatter.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments HTML formatter tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import re
+import unittest
+import StringIO
+import tempfile
+from os.path import join, dirname, isfile
+
+from pygments.lexers import PythonLexer
+from pygments.formatters import HtmlFormatter, NullFormatter
+from pygments.formatters.html import escape_html
+from pygments.util import uni_open
+
+import support
+
+TESTFILE, TESTDIR = support.location(__file__)
+
+fp = uni_open(TESTFILE, encoding='utf-8')
+try:
+    tokensource = list(PythonLexer().get_tokens(fp.read()))
+finally:
+    fp.close()
+
+
+class HtmlFormatterTest(unittest.TestCase):
+    def test_correct_output(self):
+        hfmt = HtmlFormatter(nowrap=True)
+        houtfile = StringIO.StringIO()
+        hfmt.format(tokensource, houtfile)
+
+        nfmt = NullFormatter()
+        noutfile = StringIO.StringIO()
+        nfmt.format(tokensource, noutfile)
+
+        stripped_html = re.sub('<.*?>', '', houtfile.getvalue())
+        escaped_text = escape_html(noutfile.getvalue())
+        self.assertEqual(stripped_html, escaped_text)
+
+    def test_external_css(self):
+        # test correct behavior
+        # CSS should be in /tmp directory
+        fmt1 = HtmlFormatter(full=True, cssfile='fmt1.css', outencoding='utf-8')
+        # CSS should be in TESTDIR (TESTDIR is absolute)
+        fmt2 = HtmlFormatter(full=True, cssfile=join(TESTDIR, 'fmt2.css'),
+                             outencoding='utf-8')
+        tfile = tempfile.NamedTemporaryFile(suffix='.html')
+        fmt1.format(tokensource, tfile)
+        try:
+            fmt2.format(tokensource, tfile)
+            self.assertTrue(isfile(join(TESTDIR, 'fmt2.css')))
+        except IOError:
+            # test directory not writable
+            pass
+        tfile.close()
+
+        self.assertTrue(isfile(join(dirname(tfile.name), 'fmt1.css')))
+        os.unlink(join(dirname(tfile.name), 'fmt1.css'))
+        try:
+            os.unlink(join(TESTDIR, 'fmt2.css'))
+        except OSError:
+            pass
+
+    def test_all_options(self):
+        for optdict in [dict(nowrap=True),
+                        dict(linenos=True),
+                        dict(linenos=True, full=True),
+                        dict(linenos=True, full=True, noclasses=True)]:
+
+            outfile = StringIO.StringIO()
+            fmt = HtmlFormatter(**optdict)
+            fmt.format(tokensource, outfile)
+
+    def test_linenos(self):
+        optdict = dict(linenos=True)
+        outfile = StringIO.StringIO()
+        fmt = HtmlFormatter(**optdict)
+        fmt.format(tokensource, outfile)
+        html = outfile.getvalue()
+        self.assertTrue(re.search("<pre>\s+1\s+2\s+3", html))
+
+    def test_linenos_with_startnum(self):
+        optdict = dict(linenos=True, linenostart=5)
+        outfile = StringIO.StringIO()
+        fmt = HtmlFormatter(**optdict)
+        fmt.format(tokensource, outfile)
+        html = outfile.getvalue()
+        self.assertTrue(re.search("<pre>\s+5\s+6\s+7", html))
+
+    def test_lineanchors(self):
+        optdict = dict(lineanchors="foo")
+        outfile = StringIO.StringIO()
+        fmt = HtmlFormatter(**optdict)
+        fmt.format(tokensource, outfile)
+        html = outfile.getvalue()
+        self.assertTrue(re.search("<pre><a name=\"foo-1\">", html))
+
+    def test_lineanchors_with_startnum(self):
+        optdict = dict(lineanchors="foo", linenostart=5)
+        outfile = StringIO.StringIO()
+        fmt = HtmlFormatter(**optdict)
+        fmt.format(tokensource, outfile)
+        html = outfile.getvalue()
+        self.assertTrue(re.search("<pre><a name=\"foo-5\">", html))
+
+    def test_valid_output(self):
+        # test all available wrappers
+        fmt = HtmlFormatter(full=True, linenos=True, noclasses=True,
+                            outencoding='utf-8')
+
+        handle, pathname = tempfile.mkstemp('.html')
+        tfile = os.fdopen(handle, 'w+b')
+        fmt.format(tokensource, tfile)
+        tfile.close()
+        catname = os.path.join(TESTDIR, 'dtds', 'HTML4.soc')
+        try:
+            import subprocess
+            po = subprocess.Popen(['nsgmls', '-s', '-c', catname, pathname],
+                                  stdout=subprocess.PIPE)
+            ret = po.wait()
+            output = po.stdout.read()
+            po.stdout.close()
+        except OSError:
+            # nsgmls not available
+            pass
+        else:
+            if ret:
+                print output
+            self.assertFalse(ret, 'nsgmls run reported errors')
+
+        os.unlink(pathname)
+
+    def test_get_style_defs(self):
+        fmt = HtmlFormatter()
+        sd = fmt.get_style_defs()
+        self.assertTrue(sd.startswith('.'))
+
+        fmt = HtmlFormatter(cssclass='foo')
+        sd = fmt.get_style_defs()
+        self.assertTrue(sd.startswith('.foo'))
+        sd = fmt.get_style_defs('.bar')
+        self.assertTrue(sd.startswith('.bar'))
+        sd = fmt.get_style_defs(['.bar', '.baz'])
+        fl = sd.splitlines()[0]
+        self.assertTrue('.bar' in fl and '.baz' in fl)
+
+    def test_unicode_options(self):
+        fmt = HtmlFormatter(title=u'Föö',
+                            cssclass=u'bär',
+                            cssstyles=u'div:before { content: \'bäz\' }',
+                            encoding='utf-8')
+        handle, pathname = tempfile.mkstemp('.html')
+        tfile = os.fdopen(handle, 'w+b')
+        fmt.format(tokensource, tfile)
+        tfile.close()
+
+    def test_ctags(self):
+        try:
+            import ctags
+        except ImportError:
+            # we can't check without the ctags module, but at least check the exception
+            self.assertRaises(RuntimeError, HtmlFormatter, tagsfile='support/tags')
+        else:
+            # this tagfile says that test_ctags() is on line 165, even if it isn't
+            # anymore in the actual source
+            fmt = HtmlFormatter(tagsfile='support/tags', lineanchors='L',
+                                tagurlformat='%(fname)s%(fext)s')
+            outfile = StringIO.StringIO()
+            fmt.format(tokensource, outfile)
+            self.assertTrue('<a href="test_html_formatter.py#L-165">test_ctags</a>'
+                            in outfile.getvalue())

data/vendor/pygments-main/tests/test_latex_formatter.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments LaTeX formatter tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import unittest
+import tempfile
+
+from pygments.formatters import LatexFormatter
+from pygments.lexers import PythonLexer
+
+import support
+
+TESTFILE, TESTDIR = support.location(__file__)
+
+
+class LatexFormatterTest(unittest.TestCase):
+
+    def test_valid_output(self):
+        fp = open(TESTFILE)
+        try:
+            tokensource = list(PythonLexer().get_tokens(fp.read()))
+        finally:
+            fp.close()
+        fmt = LatexFormatter(full=True, encoding='latin1')
+
+        handle, pathname = tempfile.mkstemp('.tex')
+        # place all output files in /tmp too
+        old_wd = os.getcwd()
+        os.chdir(os.path.dirname(pathname))
+        tfile = os.fdopen(handle, 'wb')
+        fmt.format(tokensource, tfile)
+        tfile.close()
+        try:
+            import subprocess
+            po = subprocess.Popen(['latex', '-interaction=nonstopmode',
+                                   pathname], stdout=subprocess.PIPE)
+            ret = po.wait()
+            output = po.stdout.read()
+            po.stdout.close()
+        except OSError:
+            # latex not available
+            pass
+        else:
+            if ret:
+                print output
+            self.assertFalse(ret, 'latex run reported errors')
+
+        os.unlink(pathname)
+        os.chdir(old_wd)

data/vendor/pygments-main/tests/test_lexers_other.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+    Tests for other lexers
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import glob
+import os
+import unittest
+
+from pygments.lexers import guess_lexer
+from pygments.lexers.other import RexxLexer
+
+
+def _exampleFilePath(filename):
+    return os.path.join(os.path.dirname(__file__), 'examplefiles', filename)
+
+
+class AnalyseTextTest(unittest.TestCase):
+    def _testCanRecognizeAndGuessExampleFiles(self, lexer):
+        assert lexer is not None
+
+        for pattern in lexer.filenames:
+            exampleFilesPattern = _exampleFilePath(pattern)
+            for exampleFilePath in glob.glob(exampleFilesPattern):
+                exampleFile = open(exampleFilePath, 'rb')
+                try:
+                    text = exampleFile.read()
+                    probability = lexer.analyse_text(text)
+                    self.assertTrue(probability > 0,
+                        '%s must recognize %r' % (
+                        lexer.name, exampleFilePath))
+                    guessedLexer = guess_lexer(text)
+                    self.assertEqual(guessedLexer.name, lexer.name)
+                finally:
+                    exampleFile.close()
+
+    def testCanRecognizeAndGuessExampleFiles(self):
+        self._testCanRecognizeAndGuessExampleFiles(RexxLexer)
+
+
+class RexxLexerTest(unittest.TestCase):
+    def testCanGuessFromText(self):
+        self.assertAlmostEqual(0.01,
+            RexxLexer.analyse_text('/* */'))
+        self.assertAlmostEqual(1.0,
+            RexxLexer.analyse_text('''/* Rexx */
+                say "hello world"'''))
+        self.assertLess(0.5,
+            RexxLexer.analyse_text('/* */\n'
+                'hello:pRoceduRe\n'
+                '  say "hello world"'))
+        self.assertLess(0.2,
+            RexxLexer.analyse_text('''/* */
+                if 1 > 0 then do
+                    say "ok"
+                end
+                else do
+                    say "huh?"
+                end'''))
+        self.assertLess(0.2,
+            RexxLexer.analyse_text('''/* */
+                greeting = "hello world!"
+                parse value greeting "hello" name "!"
+                say name'''))

data/vendor/pygments-main/tests/test_perllexer.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments regex lexer tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import time
+import unittest
+
+from pygments.token import String
+from pygments.lexers.agile import PerlLexer
+
+
+class RunawayRegexTest(unittest.TestCase):
+    # A previous version of the Perl lexer would spend a great deal of
+    # time backtracking when given particular strings.  These tests show that
+    # the runaway backtracking doesn't happen any more (at least for the given
+    # cases).
+
+    lexer = PerlLexer()
+
+    ### Test helpers.
+
+    def assert_single_token(self, s, token):
+        """Show that a given string generates only one token."""
+        tokens = list(self.lexer.get_tokens_unprocessed(s))
+        self.assertEqual(len(tokens), 1, tokens)
+        self.assertEqual(s, tokens[0][2])
+        self.assertEqual(token, tokens[0][1])
+
+    def assert_tokens(self, strings, expected_tokens):
+        """Show that a given string generates the expected tokens."""
+        tokens = list(self.lexer.get_tokens_unprocessed(''.join(strings)))
+        self.assertEqual(len(tokens), len(expected_tokens), tokens)
+        for index, s in enumerate(strings):
+            self.assertEqual(s, tokens[index][2])
+            self.assertEqual(expected_tokens[index], tokens[index][1])
+
+    def assert_fast_tokenization(self, s):
+        """Show that a given string is tokenized quickly."""
+        start = time.time()
+        tokens = list(self.lexer.get_tokens_unprocessed(s))
+        end = time.time()
+        # Isn't 10 seconds kind of a long time?  Yes, but we don't want false
+        # positives when the tests are starved for CPU time.
+        if end-start > 10:
+            self.fail('tokenization took too long')
+        return tokens
+
+    ### Strings.
+
+    def test_single_quote_strings(self):
+        self.assert_single_token(r"'foo\tbar\\\'baz'", String)
+        self.assert_fast_tokenization("'" + '\\'*999)
+
+    def test_double_quote_strings(self):
+        self.assert_single_token(r'"foo\tbar\\\"baz"', String)
+        self.assert_fast_tokenization('"' + '\\'*999)
+
+    def test_backtick_strings(self):
+        self.assert_single_token(r'`foo\tbar\\\`baz`', String.Backtick)
+        self.assert_fast_tokenization('`' + '\\'*999)
+
+    ### Regex matches with various delimiters.
+
+    def test_match(self):
+        self.assert_single_token(r'/aa\tbb/', String.Regex)
+        self.assert_fast_tokenization('/' + '\\'*999)
+
+    def test_match_with_slash(self):
+        self.assert_tokens(['m', '/\n\\t\\\\/'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m/xxx\n' + '\\'*999)
+
+    def test_match_with_bang(self):
+        self.assert_tokens(['m', r'!aa\t\!bb!'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m!' + '\\'*999)
+
+    def test_match_with_brace(self):
+        self.assert_tokens(['m', r'{aa\t\}bb}'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m{' + '\\'*999)
+
+    def test_match_with_angle_brackets(self):
+        self.assert_tokens(['m', r'<aa\t\>bb>'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m<' + '\\'*999)
+
+    def test_match_with_parenthesis(self):
+        self.assert_tokens(['m', r'(aa\t\)bb)'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m(' + '\\'*999)
+
+    def test_match_with_at_sign(self):
+        self.assert_tokens(['m', r'@aa\t\@bb@'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m@' + '\\'*999)
+
+    def test_match_with_percent_sign(self):
+        self.assert_tokens(['m', r'%aa\t\%bb%'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m%' + '\\'*999)
+
+    def test_match_with_dollar_sign(self):
+        self.assert_tokens(['m', r'$aa\t\$bb$'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m$' + '\\'*999)
+
+    ### Regex substitutions with various delimeters.
+
+    def test_substitution_with_slash(self):
+        self.assert_single_token('s/aaa/bbb/g', String.Regex)
+        self.assert_fast_tokenization('s/foo/' + '\\'*999)
+
+    def test_substitution_with_at_sign(self):
+        self.assert_single_token(r's@aaa@bbb@g', String.Regex)
+        self.assert_fast_tokenization('s@foo@' + '\\'*999)
+
+    def test_substitution_with_percent_sign(self):
+        self.assert_single_token(r's%aaa%bbb%g', String.Regex)
+        self.assert_fast_tokenization('s%foo%' + '\\'*999)
+
+    def test_substitution_with_brace(self):
+        self.assert_single_token(r's{aaa}', String.Regex)
+        self.assert_fast_tokenization('s{' + '\\'*999)
+
+    def test_substitution_with_angle_bracket(self):
+        self.assert_single_token(r's<aaa>', String.Regex)
+        self.assert_fast_tokenization('s<' + '\\'*999)
+
+    def test_substitution_with_angle_bracket(self):
+        self.assert_single_token(r's<aaa>', String.Regex)
+        self.assert_fast_tokenization('s<' + '\\'*999)
+
+    def test_substitution_with_square_bracket(self):
+        self.assert_single_token(r's[aaa]', String.Regex)
+        self.assert_fast_tokenization('s[' + '\\'*999)
+
+    def test_substitution_with_parenthesis(self):
+        self.assert_single_token(r's(aaa)', String.Regex)
+        self.assert_fast_tokenization('s(' + '\\'*999)

data/vendor/pygments-main/tests/test_regexlexer.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments regex lexer tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments.token import Text
+from pygments.lexer import RegexLexer
+from pygments.lexer import bygroups
+
+
+class TestLexer(RegexLexer):
+    """Test tuple state transitions including #pop."""
+    tokens = {
+        'root': [
+            ('a', Text.Root, 'rag'),
+            ('e', Text.Root),
+        ],
+        'beer': [
+            ('d', Text.Beer, ('#pop', '#pop')),
+        ],
+        'rag': [
+            ('b', Text.Rag, '#push'),
+            ('c', Text.Rag, ('#pop', 'beer')),
+        ],
+    }
+
+
+class TupleTransTest(unittest.TestCase):
+    def test(self):
+        lx = TestLexer()
+        toks = list(lx.get_tokens_unprocessed('abcde'))
+        self.assertEqual(toks,
+           [(0, Text.Root, 'a'), (1, Text.Rag, 'b'), (2, Text.Rag, 'c'),
+            (3, Text.Beer, 'd'), (4, Text.Root, 'e')])
+
+    def test_multiline(self):
+        lx = TestLexer()
+        toks = list(lx.get_tokens_unprocessed('a\ne'))
+        self.assertEqual(toks,
+           [(0, Text.Root, 'a'), (1, Text, u'\n'),
+            (2, Text.Root, 'e')])

data/vendor/pygments-main/tests/test_token.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+    Test suite for the token module
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments import token
+
+
+class TokenTest(unittest.TestCase):
+
+    def test_tokentype(self):
+        e = self.assertEqual
+
+        t = token.String
+
+        e(t.split(), [token.Token, token.Literal, token.String])
+
+        e(t.__class__, token._TokenType)
+
+    def test_functions(self):
+        self.assertTrue(token.is_token_subtype(token.String, token.String))
+        self.assertTrue(token.is_token_subtype(token.String, token.Literal))
+        self.assertFalse(token.is_token_subtype(token.Literal, token.String))
+
+        self.assertTrue(token.string_to_tokentype(token.String) is token.String)
+        self.assertTrue(token.string_to_tokentype('') is token.Token)
+        self.assertTrue(token.string_to_tokentype('String') is token.String)
+
+    def test_sanity_check(self):
+        stp = token.STANDARD_TYPES.copy()
+        stp[token.Token] = '---' # Token and Text do conflict, that is okay
+        t = {}
+        for k, v in stp.iteritems():
+            t.setdefault(v, []).append(k)
+        if len(t) == len(stp):
+            return # Okay
+
+        for k, v in t.iteritems():
+            if len(v) > 1:
+                self.fail("%r has more than one key: %r" % (k, v))

data/vendor/pygments-main/tests/test_using_api.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments tests for using()
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments.lexer import using, bygroups, this, RegexLexer
+from pygments.token import String, Text, Keyword
+
+class TestLexer(RegexLexer):
+    tokens = {
+        'root': [
+            (r'#.*',
+             using(this, state='invalid')),
+            (r'(")(.+?)(")',
+             bygroups(String, using(this, state='string'), String)),
+            (r'[^"]+', Text),
+        ],
+        'string': [
+            (r'.+', Keyword),
+        ],
+    }
+
+
+class UsingStateTest(unittest.TestCase):
+    def test_basic(self):
+        expected = [(Text, 'a'), (String, '"'), (Keyword, 'bcd'),
+                    (String, '"'), (Text, 'e\n')]
+        t = list(TestLexer().get_tokens('a"bcd"e'))
+        self.assertEqual(t, expected)
+
+    def test_error(self):
+        def gen():
+            return list(TestLexer().get_tokens('#a'))
+        self.assertRaises(KeyError, gen)