pygments.rb 0.5.2 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. data/README.md +2 -0
  2. data/lexers +0 -0
  3. data/lib/pygments/version.rb +1 -1
  4. data/test/test_pygments.rb +1 -1
  5. data/vendor/custom_lexers/github.py +15 -9
  6. data/vendor/pygments-main/AUTHORS +12 -2
  7. data/vendor/pygments-main/CHANGES +52 -2
  8. data/vendor/pygments-main/REVISION +1 -1
  9. data/vendor/pygments-main/docs/src/lexerdevelopment.txt +52 -0
  10. data/vendor/pygments-main/external/lasso-builtins-generator-9.lasso +67 -44
  11. data/vendor/pygments-main/pygmentize +1 -1
  12. data/vendor/pygments-main/pygments/filters/__init__.py +2 -2
  13. data/vendor/pygments-main/pygments/formatter.py +3 -0
  14. data/vendor/pygments-main/pygments/lexers/__init__.py +11 -0
  15. data/vendor/pygments-main/pygments/lexers/_lassobuiltins.py +2880 -3124
  16. data/vendor/pygments-main/pygments/lexers/_mapping.py +30 -20
  17. data/vendor/pygments-main/pygments/lexers/_robotframeworklexer.py +1 -1
  18. data/vendor/pygments-main/pygments/lexers/_stan_builtins.py +206 -20
  19. data/vendor/pygments-main/pygments/lexers/agile.py +378 -5
  20. data/vendor/pygments-main/pygments/lexers/asm.py +2 -2
  21. data/vendor/pygments-main/pygments/lexers/compiled.py +235 -8
  22. data/vendor/pygments-main/pygments/lexers/dotnet.py +88 -47
  23. data/vendor/pygments-main/pygments/lexers/functional.py +195 -62
  24. data/vendor/pygments-main/pygments/lexers/github.py +15 -9
  25. data/vendor/pygments-main/pygments/lexers/jvm.py +14 -11
  26. data/vendor/pygments-main/pygments/lexers/math.py +284 -18
  27. data/vendor/pygments-main/pygments/lexers/other.py +132 -21
  28. data/vendor/pygments-main/pygments/lexers/shell.py +29 -15
  29. data/vendor/pygments-main/pygments/lexers/sql.py +1 -1
  30. data/vendor/pygments-main/pygments/lexers/templates.py +8 -8
  31. data/vendor/pygments-main/pygments/lexers/text.py +59 -9
  32. data/vendor/pygments-main/pygments/lexers/web.py +832 -210
  33. data/vendor/pygments-main/pygments/modeline.py +40 -0
  34. data/vendor/pygments-main/tests/examplefiles/Deflate.fs +578 -0
  35. data/vendor/pygments-main/tests/examplefiles/Get-CommandDefinitionHtml.ps1 +66 -0
  36. data/vendor/pygments-main/tests/examplefiles/IPDispatchC.nc +104 -0
  37. data/vendor/pygments-main/tests/examplefiles/IPDispatchP.nc +671 -0
  38. data/vendor/pygments-main/tests/examplefiles/RoleQ.pm6 +23 -0
  39. data/vendor/pygments-main/tests/examplefiles/example.ceylon +29 -10
  40. data/vendor/pygments-main/tests/examplefiles/example.clay +33 -0
  41. data/vendor/pygments-main/tests/examplefiles/example.hx +142 -0
  42. data/vendor/pygments-main/tests/examplefiles/example.lagda +19 -0
  43. data/vendor/pygments-main/tests/examplefiles/example.rexx +50 -0
  44. data/vendor/pygments-main/tests/examplefiles/example.stan +86 -75
  45. data/vendor/pygments-main/tests/examplefiles/garcia-wachs.kk +40 -30
  46. data/vendor/pygments-main/tests/examplefiles/grammar-test.p6 +22 -0
  47. data/vendor/pygments-main/tests/examplefiles/objc_example.m +7 -0
  48. data/vendor/pygments-main/tests/examplefiles/py3tb_test.py3tb +4 -0
  49. data/vendor/pygments-main/tests/examplefiles/swig_java.swg +1329 -0
  50. data/vendor/pygments-main/tests/examplefiles/swig_std_vector.i +225 -0
  51. data/vendor/pygments-main/tests/examplefiles/test.agda +102 -0
  52. data/vendor/pygments-main/tests/examplefiles/test.bb +95 -0
  53. data/vendor/pygments-main/tests/examplefiles/test.ebnf +31 -0
  54. data/vendor/pygments-main/tests/examplefiles/test.p6 +252 -0
  55. data/vendor/pygments-main/tests/examplefiles/type.lisp +16 -0
  56. data/vendor/pygments-main/tests/test_basic_api.py +3 -3
  57. data/vendor/pygments-main/tests/test_lexers_other.py +68 -0
  58. metadata +21 -2
@@ -529,7 +529,7 @@ class VbNetAspxLexer(DelegatingLexer):
529
529
  # Very close to functional.OcamlLexer
530
530
  class FSharpLexer(RegexLexer):
531
531
  """
532
- For the F# language.
532
+ For the F# language (version 3.0).
533
533
 
534
534
  *New in Pygments 1.5.*
535
535
  """
@@ -540,91 +540,132 @@ class FSharpLexer(RegexLexer):
540
540
  mimetypes = ['text/x-fsharp']
541
541
 
542
542
  keywords = [
543
- 'abstract', 'and', 'as', 'assert', 'base', 'begin', 'class',
544
- 'default', 'delegate', 'do', 'do!', 'done', 'downcast',
545
- 'downto', 'elif', 'else', 'end', 'exception', 'extern',
546
- 'false', 'finally', 'for', 'fun', 'function', 'global', 'if',
547
- 'in', 'inherit', 'inline', 'interface', 'internal', 'lazy',
548
- 'let', 'let!', 'match', 'member', 'module', 'mutable',
549
- 'namespace', 'new', 'null', 'of', 'open', 'or', 'override',
550
- 'private', 'public', 'rec', 'return', 'return!', 'sig',
551
- 'static', 'struct', 'then', 'to', 'true', 'try', 'type',
552
- 'upcast', 'use', 'use!', 'val', 'void', 'when', 'while',
553
- 'with', 'yield', 'yield!'
543
+ 'abstract', 'as', 'assert', 'base', 'begin', 'class', 'default',
544
+ 'delegate', 'do!', 'do', 'done', 'downcast', 'downto', 'elif', 'else',
545
+ 'end', 'exception', 'extern', 'false', 'finally', 'for', 'function',
546
+ 'fun', 'global', 'if', 'inherit', 'inline', 'interface', 'internal',
547
+ 'in', 'lazy', 'let!', 'let', 'match', 'member', 'module', 'mutable',
548
+ 'namespace', 'new', 'null', 'of', 'open', 'override', 'private', 'public',
549
+ 'rec', 'return!', 'return', 'select', 'static', 'struct', 'then', 'to',
550
+ 'true', 'try', 'type', 'upcast', 'use!', 'use', 'val', 'void', 'when',
551
+ 'while', 'with', 'yield!', 'yield',
552
+ ]
553
+ # Reserved words; cannot hurt to color them as keywords too.
554
+ keywords += [
555
+ 'atomic', 'break', 'checked', 'component', 'const', 'constraint',
556
+ 'constructor', 'continue', 'eager', 'event', 'external', 'fixed',
557
+ 'functor', 'include', 'method', 'mixin', 'object', 'parallel',
558
+ 'process', 'protected', 'pure', 'sealed', 'tailcall', 'trait',
559
+ 'virtual', 'volatile',
554
560
  ]
555
561
  keyopts = [
556
- '!=','#','&&','&','\(','\)','\*','\+',',','-\.',
557
- '->','-','\.\.','\.','::',':=',':>',':',';;',';','<-',
558
- '<','>]','>','\?\?','\?','\[<','\[>','\[\|','\[',
559
- ']','_','`','{','\|\]','\|','}','~','<@','=','@>'
562
+ '!=', '#', '&&', '&', '\(', '\)', '\*', '\+', ',', '-\.',
563
+ '->', '-', '\.\.', '\.', '::', ':=', ':>', ':', ';;', ';', '<-',
564
+ '<\]', '<', '>\]', '>', '\?\?', '\?', '\[<', '\[\|', '\[', '\]',
565
+ '_', '`', '{', '\|\]', '\|', '}', '~', '<@@', '<@', '=', '@>', '@@>',
560
566
  ]
561
567
 
562
568
  operators = r'[!$%&*+\./:<=>?@^|~-]'
563
- word_operators = ['and', 'asr', 'land', 'lor', 'lsl', 'lxor', 'mod', 'not', 'or']
569
+ word_operators = ['and', 'or', 'not']
564
570
  prefix_syms = r'[!?~]'
565
571
  infix_syms = r'[=<>@^|&+\*/$%-]'
566
- primitives = ['unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array',
567
- 'byte', 'sbyte', 'int16', 'uint16', 'uint32', 'int64', 'uint64'
568
- 'nativeint', 'unativeint', 'decimal', 'void', 'float32', 'single',
569
- 'double']
572
+ primitives = [
573
+ 'sbyte', 'byte', 'char', 'nativeint', 'unativeint', 'float32', 'single',
574
+ 'float', 'double', 'int8', 'uint8', 'int16', 'uint16', 'int32',
575
+ 'uint32', 'int64', 'uint64', 'decimal', 'unit', 'bool', 'string',
576
+ 'list', 'exn', 'obj', 'enum',
577
+ ]
578
+
579
+ # See http://msdn.microsoft.com/en-us/library/dd233181.aspx and/or
580
+ # http://fsharp.org/about/files/spec.pdf for reference. Good luck.
570
581
 
571
582
  tokens = {
572
583
  'escape-sequence': [
573
- (r'\\[\\\"\'ntbr]', String.Escape),
584
+ (r'\\[\\\"\'ntbrafv]', String.Escape),
574
585
  (r'\\[0-9]{3}', String.Escape),
575
- (r'\\x[0-9a-fA-F]{2}', String.Escape),
586
+ (r'\\u[0-9a-fA-F]{4}', String.Escape),
587
+ (r'\\U[0-9a-fA-F]{8}', String.Escape),
576
588
  ],
577
589
  'root': [
578
590
  (r'\s+', Text),
579
- (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo),
580
- (r'\b([A-Z][A-Za-z0-9_\']*)(?=\s*\.)',
591
+ (r'\(\)|\[\]', Name.Builtin.Pseudo),
592
+ (r'\b(?<!\.)([A-Z][A-Za-z0-9_\']*)(?=\s*\.)',
581
593
  Name.Namespace, 'dotted'),
582
- (r'\b([A-Z][A-Za-z0-9_\']*)', Name.Class),
594
+ (r'\b([A-Z][A-Za-z0-9_\']*)', Name),
595
+ (r'///.*?\n', String.Doc),
583
596
  (r'//.*?\n', Comment.Single),
584
597
  (r'\(\*(?!\))', Comment, 'comment'),
598
+
599
+ (r'@"', String, 'lstring'),
600
+ (r'"""', String, 'tqs'),
601
+ (r'"', String, 'string'),
602
+
603
+ (r'\b(open|module)(\s+)([a-zA-Z0-9_.]+)',
604
+ bygroups(Keyword, Text, Name.Namespace)),
605
+ (r'\b(let!?)(\s+)([a-zA-Z0-9_]+)',
606
+ bygroups(Keyword, Text, Name.Variable)),
607
+ (r'\b(type)(\s+)([a-zA-Z0-9_]+)',
608
+ bygroups(Keyword, Text, Name.Class)),
609
+ (r'\b(member|override)(\s+)([a-zA-Z0-9_]+)(\.)([a-zA-Z0-9_]+)',
610
+ bygroups(Keyword, Text, Name, Punctuation, Name.Function)),
585
611
  (r'\b(%s)\b' % '|'.join(keywords), Keyword),
586
612
  (r'(%s)' % '|'.join(keyopts), Operator),
587
613
  (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
588
614
  (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word),
589
615
  (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type),
590
-
591
- (r'#[ \t]*(if|endif|else|line|nowarn|light)\b.*?\n',
616
+ (r'#[ \t]*(if|endif|else|line|nowarn|light|\d+)\b.*?\n',
592
617
  Comment.Preproc),
593
618
 
594
619
  (r"[^\W\d][\w']*", Name),
595
620
 
596
- (r'\d[\d_]*', Number.Integer),
597
- (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
598
- (r'0[oO][0-7][0-7_]*', Number.Oct),
599
- (r'0[bB][01][01_]*', Number.Binary),
600
- (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
621
+ (r'\d[\d_]*[uU]?[yslLnQRZINGmM]?', Number.Integer),
622
+ (r'0[xX][\da-fA-F][\da-fA-F_]*[uU]?[yslLn]?[fF]?', Number.Hex),
623
+ (r'0[oO][0-7][0-7_]*[uU]?[yslLn]?', Number.Oct),
624
+ (r'0[bB][01][01_]*[uU]?[yslLn]?', Number.Binary),
625
+ (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)[fFmM]?',
626
+ Number.Float),
601
627
 
602
- (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
628
+ (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'B?",
603
629
  String.Char),
604
630
  (r"'.'", String.Char),
605
631
  (r"'", Keyword), # a stray quote is another syntax element
606
632
 
607
- (r'"', String.Double, 'string'),
608
-
609
633
  (r'[~?][a-z][\w\']*:', Name.Variable),
610
634
  ],
635
+ 'dotted': [
636
+ (r'\s+', Text),
637
+ (r'\.', Punctuation),
638
+ (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace),
639
+ (r'[A-Z][A-Za-z0-9_\']*', Name, '#pop'),
640
+ (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'),
641
+ ],
611
642
  'comment': [
612
- (r'[^(*)]+', Comment),
643
+ (r'[^(*)@"]+', Comment),
613
644
  (r'\(\*', Comment, '#push'),
614
645
  (r'\*\)', Comment, '#pop'),
615
- (r'[(*)]', Comment),
646
+ # comments cannot be closed within strings in comments
647
+ (r'@"', String, 'lstring'),
648
+ (r'"""', String, 'tqs'),
649
+ (r'"', String, 'string'),
650
+ (r'[(*)@]', Comment),
616
651
  ],
617
652
  'string': [
618
- (r'[^\\"]+', String.Double),
653
+ (r'[^\\"]+', String),
619
654
  include('escape-sequence'),
620
- (r'\\\n', String.Double),
621
- (r'"', String.Double, '#pop'),
655
+ (r'\\\n', String),
656
+ (r'\n', String), # newlines are allowed in any string
657
+ (r'"B?', String, '#pop'),
622
658
  ],
623
- 'dotted': [
624
- (r'\s+', Text),
625
- (r'\.', Punctuation),
626
- (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace),
627
- (r'[A-Z][A-Za-z0-9_\']*', Name.Class, '#pop'),
628
- (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'),
659
+ 'lstring': [
660
+ (r'[^"]+', String),
661
+ (r'\n', String),
662
+ (r'""', String),
663
+ (r'"B?', String, '#pop'),
664
+ ],
665
+ 'tqs': [
666
+ (r'[^"]+', String),
667
+ (r'\n', String),
668
+ (r'"""B?', String, '#pop'),
669
+ (r'"', String),
629
670
  ],
630
671
  }
@@ -16,9 +16,13 @@ from pygments.token import Text, Comment, Operator, Keyword, Name, \
16
16
  String, Number, Punctuation, Literal, Generic, Error
17
17
 
18
18
  __all__ = ['RacketLexer', 'SchemeLexer', 'CommonLispLexer', 'HaskellLexer',
19
- 'LiterateHaskellLexer', 'SMLLexer', 'OcamlLexer', 'ErlangLexer',
20
- 'ErlangShellLexer', 'OpaLexer', 'CoqLexer', 'NewLispLexer',
21
- 'ElixirLexer', 'ElixirConsoleLexer', 'KokaLexer']
19
+ 'AgdaLexer', 'LiterateHaskellLexer', 'LiterateAgdaLexer',
20
+ 'SMLLexer', 'OcamlLexer', 'ErlangLexer', 'ErlangShellLexer',
21
+ 'OpaLexer', 'CoqLexer', 'NewLispLexer', 'ElixirLexer',
22
+ 'ElixirConsoleLexer', 'KokaLexer']
23
+
24
+
25
+ line_re = re.compile('.*?\n')
22
26
 
23
27
 
24
28
  class RacketLexer(RegexLexer):
@@ -719,7 +723,7 @@ class CommonLispLexer(RegexLexer):
719
723
  *New in Pygments 0.9.*
720
724
  """
721
725
  name = 'Common Lisp'
722
- aliases = ['common-lisp', 'cl']
726
+ aliases = ['common-lisp', 'cl', 'lisp']
723
727
  filenames = ['*.cl', '*.lisp', '*.el'] # use for Elisp too
724
728
  mimetypes = ['text/x-common-lisp']
725
729
 
@@ -808,6 +812,8 @@ class CommonLispLexer(RegexLexer):
808
812
  (r'"(\\.|\\\n|[^"\\])*"', String),
809
813
  # quoting
810
814
  (r":" + symbol, String.Symbol),
815
+ (r"::" + symbol, String.Symbol),
816
+ (r":#" + symbol, String.Symbol),
811
817
  (r"'" + symbol, String.Symbol),
812
818
  (r"'", Operator),
813
819
  (r"`", Operator),
@@ -979,6 +985,8 @@ class HaskellLexer(RegexLexer):
979
985
  (r'\(', Punctuation, ('funclist', 'funclist')),
980
986
  (r'\)', Punctuation, '#pop:2'),
981
987
  ],
988
+ # NOTE: the next four states are shared in the AgdaLexer; make sure
989
+ # any change is compatible with Agda as well or copy over and change
982
990
  'comment': [
983
991
  # Multiline Comments
984
992
  (r'[^-{}]+', Comment.Multiline),
@@ -1009,12 +1017,78 @@ class HaskellLexer(RegexLexer):
1009
1017
  }
1010
1018
 
1011
1019
 
1012
- line_re = re.compile('.*?\n')
1013
- bird_re = re.compile(r'(>[ \t]*)(.*\n)')
1020
+ class AgdaLexer(RegexLexer):
1021
+ """
1022
+ For the `Agda <http://wiki.portal.chalmers.se/agda/pmwiki.php>`_
1023
+ dependently typed functional programming language and proof assistant.
1014
1024
 
1015
- class LiterateHaskellLexer(Lexer):
1025
+ *New in Pygments 1.7.*
1016
1026
  """
1017
- For Literate Haskell (Bird-style or LaTeX) source.
1027
+
1028
+ name = 'Agda'
1029
+ aliases = ['agda']
1030
+ filenames = ['*.agda']
1031
+ mimetypes = ['text/x-agda']
1032
+
1033
+ reserved = ['abstract', 'codata', 'coinductive', 'constructor', 'data',
1034
+ 'field', 'forall', 'hiding', 'in', 'inductive', 'infix',
1035
+ 'infixl', 'infixr', 'let', 'open', 'pattern', 'primitive',
1036
+ 'private', 'mutual', 'quote', 'quoteGoal', 'quoteTerm',
1037
+ 'record', 'syntax', 'rewrite', 'unquote', 'using', 'where',
1038
+ 'with']
1039
+
1040
+ tokens = {
1041
+ 'root': [
1042
+ # Declaration
1043
+ (r'^(\s*)([^\s\(\)\{\}]+)(\s*)(:)(\s*)',
1044
+ bygroups(Text, Name.Function, Text, Operator.Word, Text)),
1045
+ # Comments
1046
+ (r'--(?![!#$%&*+./<=>?@\^|_~:\\]).*?$', Comment.Single),
1047
+ (r'{-', Comment.Multiline, 'comment'),
1048
+ # Holes
1049
+ (r'{!', Comment.Directive, 'hole'),
1050
+ # Lexemes:
1051
+ # Identifiers
1052
+ (ur'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved),
1053
+ (r'(import|module)(\s+)', bygroups(Keyword.Reserved, Text), 'module'),
1054
+ (r'\b(Set|Prop)\b', Keyword.Type),
1055
+ # Special Symbols
1056
+ (r'(\(|\)|\{|\})', Operator),
1057
+ (ur'(\.{1,3}|\||[\u039B]|[\u2200]|[\u2192]|:|=|->)', Operator.Word),
1058
+ # Numbers
1059
+ (r'\d+[eE][+-]?\d+', Number.Float),
1060
+ (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float),
1061
+ (r'0[xX][\da-fA-F]+', Number.Hex),
1062
+ (r'\d+', Number.Integer),
1063
+ # Strings
1064
+ (r"'", String.Char, 'character'),
1065
+ (r'"', String, 'string'),
1066
+ (r'[^\s\(\)\{\}]+', Text),
1067
+ (r'\s+?', Text), # Whitespace
1068
+ ],
1069
+ 'hole': [
1070
+ # Holes
1071
+ (r'[^!{}]+', Comment.Directive),
1072
+ (r'{!', Comment.Directive, '#push'),
1073
+ (r'!}', Comment.Directive, '#pop'),
1074
+ (r'[!{}]', Comment.Directive),
1075
+ ],
1076
+ 'module': [
1077
+ (r'{-', Comment.Multiline, 'comment'),
1078
+ (r'[a-zA-Z][a-zA-Z0-9_.]*', Name, '#pop'),
1079
+ (r'[^a-zA-Z]*', Text)
1080
+ ],
1081
+ 'comment': HaskellLexer.tokens['comment'],
1082
+ 'character': HaskellLexer.tokens['character'],
1083
+ 'string': HaskellLexer.tokens['string'],
1084
+ 'escape': HaskellLexer.tokens['escape']
1085
+ }
1086
+
1087
+
1088
+ class LiterateLexer(Lexer):
1089
+ """
1090
+ Base class for lexers of literate file formats based on LaTeX or Bird-style
1091
+ (prefixing each code line with ">").
1018
1092
 
1019
1093
  Additional options accepted:
1020
1094
 
@@ -1022,17 +1096,15 @@ class LiterateHaskellLexer(Lexer):
1022
1096
  If given, must be ``"bird"`` or ``"latex"``. If not given, the style
1023
1097
  is autodetected: if the first non-whitespace character in the source
1024
1098
  is a backslash or percent character, LaTeX is assumed, else Bird.
1025
-
1026
- *New in Pygments 0.9.*
1027
1099
  """
1028
- name = 'Literate Haskell'
1029
- aliases = ['lhs', 'literate-haskell']
1030
- filenames = ['*.lhs']
1031
- mimetypes = ['text/x-literate-haskell']
1032
1100
 
1033
- def get_tokens_unprocessed(self, text):
1034
- hslexer = HaskellLexer(**self.options)
1101
+ bird_re = re.compile(r'(>[ \t]*)(.*\n)')
1102
+
1103
+ def __init__(self, baselexer, **options):
1104
+ self.baselexer = baselexer
1105
+ Lexer.__init__(self, **options)
1035
1106
 
1107
+ def get_tokens_unprocessed(self, text):
1036
1108
  style = self.options.get('litstyle')
1037
1109
  if style is None:
1038
1110
  style = (text.lstrip()[0:1] in '%\\') and 'latex' or 'bird'
@@ -1043,7 +1115,7 @@ class LiterateHaskellLexer(Lexer):
1043
1115
  # bird-style
1044
1116
  for match in line_re.finditer(text):
1045
1117
  line = match.group()
1046
- m = bird_re.match(line)
1118
+ m = self.bird_re.match(line)
1047
1119
  if m:
1048
1120
  insertions.append((len(code),
1049
1121
  [(0, Comment.Special, m.group(1))]))
@@ -1054,7 +1126,6 @@ class LiterateHaskellLexer(Lexer):
1054
1126
  # latex-style
1055
1127
  from pygments.lexers.text import TexLexer
1056
1128
  lxlexer = TexLexer(**self.options)
1057
-
1058
1129
  codelines = 0
1059
1130
  latex = ''
1060
1131
  for match in line_re.finditer(text):
@@ -1075,10 +1146,56 @@ class LiterateHaskellLexer(Lexer):
1075
1146
  latex += line
1076
1147
  insertions.append((len(code),
1077
1148
  list(lxlexer.get_tokens_unprocessed(latex))))
1078
- for item in do_insertions(insertions, hslexer.get_tokens_unprocessed(code)):
1149
+ for item in do_insertions(insertions, self.baselexer.get_tokens_unprocessed(code)):
1079
1150
  yield item
1080
1151
 
1081
1152
 
1153
+ class LiterateHaskellLexer(LiterateLexer):
1154
+ """
1155
+ For Literate Haskell (Bird-style or LaTeX) source.
1156
+
1157
+ Additional options accepted:
1158
+
1159
+ `litstyle`
1160
+ If given, must be ``"bird"`` or ``"latex"``. If not given, the style
1161
+ is autodetected: if the first non-whitespace character in the source
1162
+ is a backslash or percent character, LaTeX is assumed, else Bird.
1163
+
1164
+ *New in Pygments 0.9.*
1165
+ """
1166
+ name = 'Literate Haskell'
1167
+ aliases = ['lhs', 'literate-haskell', 'lhaskell']
1168
+ filenames = ['*.lhs']
1169
+ mimetypes = ['text/x-literate-haskell']
1170
+
1171
+ def __init__(self, **options):
1172
+ hslexer = HaskellLexer(**options)
1173
+ LiterateLexer.__init__(self, hslexer, **options)
1174
+
1175
+
1176
+ class LiterateAgdaLexer(LiterateLexer):
1177
+ """
1178
+ For Literate Agda source.
1179
+
1180
+ Additional options accepted:
1181
+
1182
+ `litstyle`
1183
+ If given, must be ``"bird"`` or ``"latex"``. If not given, the style
1184
+ is autodetected: if the first non-whitespace character in the source
1185
+ is a backslash or percent character, LaTeX is assumed, else Bird.
1186
+
1187
+ *New in Pygments 1.7.*
1188
+ """
1189
+ name = 'Literate Agda'
1190
+ aliases = ['lagda', 'literate-agda']
1191
+ filenames = ['*.lagda']
1192
+ mimetypes = ['text/x-literate-agda']
1193
+
1194
+ def __init__(self, **options):
1195
+ agdalexer = AgdaLexer(**options)
1196
+ LiterateLexer.__init__(self, agdalexer, litstyle='latex', **options)
1197
+
1198
+
1082
1199
  class SMLLexer(RegexLexer):
1083
1200
  """
1084
1201
  For the Standard ML language.
@@ -1663,9 +1780,10 @@ class OpaLexer(RegexLexer):
1663
1780
  # but if you color only real keywords, you might just
1664
1781
  # as well not color anything
1665
1782
  keywords = [
1666
- 'and', 'as', 'begin', 'css', 'database', 'db', 'do', 'else', 'end',
1667
- 'external', 'forall', 'if', 'import', 'match', 'package', 'parser',
1668
- 'rec', 'server', 'then', 'type', 'val', 'with', 'xml_parser',
1783
+ 'and', 'as', 'begin', 'case', 'client', 'css', 'database', 'db', 'do',
1784
+ 'else', 'end', 'external', 'forall', 'function', 'if', 'import',
1785
+ 'match', 'module', 'or', 'package', 'parser', 'rec', 'server', 'then',
1786
+ 'type', 'val', 'with', 'xml_parser',
1669
1787
  ]
1670
1788
 
1671
1789
  # matches both stuff and `stuff`
@@ -2399,7 +2517,7 @@ class ElixirConsoleLexer(Lexer):
2399
2517
 
2400
2518
  class KokaLexer(RegexLexer):
2401
2519
  """
2402
- Lexer for the `Koka <http://research.microsoft.com/en-us/projects/koka/>`_
2520
+ Lexer for the `Koka <http://koka.codeplex.com>`_
2403
2521
  language.
2404
2522
 
2405
2523
  *New in Pygments 1.6.*
@@ -2411,7 +2529,7 @@ class KokaLexer(RegexLexer):
2411
2529
  mimetypes = ['text/x-koka']
2412
2530
 
2413
2531
  keywords = [
2414
- 'infix', 'infixr', 'infixl', 'prefix', 'postfix',
2532
+ 'infix', 'infixr', 'infixl',
2415
2533
  'type', 'cotype', 'rectype', 'alias',
2416
2534
  'struct', 'con',
2417
2535
  'fun', 'function', 'val', 'var',
@@ -2450,7 +2568,12 @@ class KokaLexer(RegexLexer):
2450
2568
  sboundary = '(?!'+symbols+')'
2451
2569
 
2452
2570
  # name boundary: a keyword should not be followed by any of these
2453
- boundary = '(?![a-zA-Z0-9_\\-])'
2571
+ boundary = '(?![\w/])'
2572
+
2573
+ # koka token abstractions
2574
+ tokenType = Name.Attribute
2575
+ tokenTypeDef = Name.Class
2576
+ tokenConstructor = Generic.Emph
2454
2577
 
2455
2578
  # main lexer
2456
2579
  tokens = {
@@ -2458,41 +2581,51 @@ class KokaLexer(RegexLexer):
2458
2581
  include('whitespace'),
2459
2582
 
2460
2583
  # go into type mode
2461
- (r'::?' + sboundary, Keyword.Type, 'type'),
2462
- (r'alias' + boundary, Keyword, 'alias-type'),
2463
- (r'struct' + boundary, Keyword, 'struct-type'),
2464
- (r'(%s)' % '|'.join(typeStartKeywords) + boundary, Keyword, 'type'),
2584
+ (r'::?' + sboundary, tokenType, 'type'),
2585
+ (r'(alias)(\s+)([a-z]\w*)?', bygroups(Keyword, Text, tokenTypeDef),
2586
+ 'alias-type'),
2587
+ (r'(struct)(\s+)([a-z]\w*)?', bygroups(Keyword, Text, tokenTypeDef),
2588
+ 'struct-type'),
2589
+ ((r'(%s)' % '|'.join(typeStartKeywords)) +
2590
+ r'(\s+)([a-z]\w*)?', bygroups(Keyword, Text, tokenTypeDef),
2591
+ 'type'),
2465
2592
 
2466
2593
  # special sequences of tokens (we use ?: for non-capturing group as
2467
2594
  # required by 'bygroups')
2468
- (r'(module)(\s*)((?:interface)?)(\s*)'
2469
- r'((?:[a-z](?:[a-zA-Z0-9_]|\-[a-zA-Z])*\.)*'
2470
- r'[a-z](?:[a-zA-Z0-9_]|\-[a-zA-Z])*)',
2471
- bygroups(Keyword, Text, Keyword, Text, Name.Namespace)),
2472
- (r'(import)(\s+)((?:[a-z](?:[a-zA-Z0-9_]|\-[a-zA-Z])*\.)*[a-z]'
2473
- r'(?:[a-zA-Z0-9_]|\-[a-zA-Z])*)(\s*)((?:as)?)'
2474
- r'((?:[A-Z](?:[a-zA-Z0-9_]|\-[a-zA-Z])*)?)',
2475
- bygroups(Keyword, Text, Name.Namespace, Text, Keyword,
2476
- Name.Namespace)),
2595
+ (r'(module)(\s+)(interface\s+)?((?:[a-z]\w*/)*[a-z]\w*)',
2596
+ bygroups(Keyword, Text, Keyword, Name.Namespace)),
2597
+ (r'(import)(\s+)((?:[a-z]\w*/)*[a-z]\w*)'
2598
+ r'(?:(\s*)(=)(\s*)((?:qualified\s*)?)'
2599
+ r'((?:[a-z]\w*/)*[a-z]\w*))?',
2600
+ bygroups(Keyword, Text, Name.Namespace, Text, Keyword, Text,
2601
+ Keyword, Name.Namespace)),
2602
+
2603
+ (r'(^(?:(?:public|private)\s*)?(?:function|fun|val))'
2604
+ r'(\s+)([a-z]\w*|\((?:' + symbols + r'|/)\))',
2605
+ bygroups(Keyword, Text, Name.Function)),
2606
+ (r'(^(?:(?:public|private)\s*)?external)(\s+)(inline\s+)?'
2607
+ r'([a-z]\w*|\((?:' + symbols + r'|/)\))',
2608
+ bygroups(Keyword, Text, Keyword, Name.Function)),
2477
2609
 
2478
2610
  # keywords
2479
2611
  (r'(%s)' % '|'.join(typekeywords) + boundary, Keyword.Type),
2480
2612
  (r'(%s)' % '|'.join(keywords) + boundary, Keyword),
2481
2613
  (r'(%s)' % '|'.join(builtin) + boundary, Keyword.Pseudo),
2482
- (r'::|:=|\->|[=\.:]' + sboundary, Keyword),
2483
- (r'\-' + sboundary, Generic.Strong),
2614
+ (r'::?|:=|\->|[=\.]' + sboundary, Keyword),
2484
2615
 
2485
2616
  # names
2486
- (r'[A-Z]([a-zA-Z0-9_]|\-[a-zA-Z])*(?=\.)', Name.Namespace),
2487
- (r'[A-Z]([a-zA-Z0-9_]|\-[a-zA-Z])*(?!\.)', Name.Class),
2488
- (r'[a-z]([a-zA-Z0-9_]|\-[a-zA-Z])*', Name),
2489
- (r'_([a-zA-Z0-9_]|\-[a-zA-Z])*', Name.Variable),
2617
+ (r'((?:[a-z]\w*/)*)([A-Z]\w*)',
2618
+ bygroups(Name.Namespace, tokenConstructor)),
2619
+ (r'((?:[a-z]\w*/)*)([a-z]\w*)', bygroups(Name.Namespace, Name)),
2620
+ (r'((?:[a-z]\w*/)*)(\((?:' + symbols + r'|/)\))',
2621
+ bygroups(Name.Namespace, Name)),
2622
+ (r'_\w*', Name.Variable),
2490
2623
 
2491
2624
  # literal string
2492
2625
  (r'@"', String.Double, 'litstring'),
2493
2626
 
2494
2627
  # operators
2495
- (symbols, Operator),
2628
+ (symbols + "|/(?![\*/])", Operator),
2496
2629
  (r'`', Operator),
2497
2630
  (r'[\{\}\(\)\[\];,]', Punctuation),
2498
2631
 
@@ -2519,17 +2652,17 @@ class KokaLexer(RegexLexer):
2519
2652
 
2520
2653
  # type started by colon
2521
2654
  'type': [
2522
- (r'[\(\[<]', Keyword.Type, 'type-nested'),
2655
+ (r'[\(\[<]', tokenType, 'type-nested'),
2523
2656
  include('type-content')
2524
2657
  ],
2525
2658
 
2526
2659
  # type nested in brackets: can contain parameters, comma etc.
2527
2660
  'type-nested': [
2528
- (r'[\)\]>]', Keyword.Type, '#pop'),
2529
- (r'[\(\[<]', Keyword.Type, 'type-nested'),
2530
- (r',', Keyword.Type),
2531
- (r'([a-z](?:[a-zA-Z0-9_]|\-[a-zA-Z])*)(\s*)(:)(?!:)',
2532
- bygroups(Name.Variable,Text,Keyword.Type)), # parameter name
2661
+ (r'[\)\]>]', tokenType, '#pop'),
2662
+ (r'[\(\[<]', tokenType, 'type-nested'),
2663
+ (r',', tokenType),
2664
+ (r'([a-z]\w*)(\s*)(:)(?!:)',
2665
+ bygroups(Name, Text, tokenType)), # parameter name
2533
2666
  include('type-content')
2534
2667
  ],
2535
2668
 
@@ -2538,23 +2671,23 @@ class KokaLexer(RegexLexer):
2538
2671
  include('whitespace'),
2539
2672
 
2540
2673
  # keywords
2541
- (r'(%s)' % '|'.join(typekeywords) + boundary, Keyword.Type),
2674
+ (r'(%s)' % '|'.join(typekeywords) + boundary, Keyword),
2542
2675
  (r'(?=((%s)' % '|'.join(keywords) + boundary + '))',
2543
2676
  Keyword, '#pop'), # need to match because names overlap...
2544
2677
 
2545
2678
  # kinds
2546
- (r'[EPH]' + boundary, Keyword.Type),
2547
- (r'[*!]', Keyword.Type),
2679
+ (r'[EPHVX]' + boundary, tokenType),
2548
2680
 
2549
2681
  # type names
2550
- (r'[A-Z]([a-zA-Z0-9_]|\-[a-zA-Z])*(?=\.)', Name.Namespace),
2551
- (r'[A-Z]([a-zA-Z0-9_]|\-[a-zA-Z])*(?!\.)', Name.Class),
2552
- (r'[a-z][0-9]*(?![a-zA-Z_\-])', Keyword.Type), # Generic.Emph
2553
- (r'_([a-zA-Z0-9_]|\-[a-zA-Z])*', Keyword.Type), # Generic.Emph
2554
- (r'[a-z]([a-zA-Z0-9_]|\-[a-zA-Z])*', Keyword.Type),
2682
+ (r'[a-z][0-9]*(?![\w/])', tokenType ),
2683
+ (r'_\w*', tokenType.Variable), # Generic.Emph
2684
+ (r'((?:[a-z]\w*/)*)([A-Z]\w*)',
2685
+ bygroups(Name.Namespace, tokenType)),
2686
+ (r'((?:[a-z]\w*/)*)([a-z]\w+)',
2687
+ bygroups(Name.Namespace, tokenType)),
2555
2688
 
2556
2689
  # type keyword operators
2557
- (r'::|\->|[\.:|]', Keyword.Type),
2690
+ (r'::|\->|[\.:|]', tokenType),
2558
2691
 
2559
2692
  #catchall
2560
2693
  (r'', Text, '#pop')
@@ -2562,6 +2695,7 @@ class KokaLexer(RegexLexer):
2562
2695
 
2563
2696
  # comments and literals
2564
2697
  'whitespace': [
2698
+ (r'\n\s*#.*$', Comment.Preproc),
2565
2699
  (r'\s+', Text),
2566
2700
  (r'/\*', Comment.Multiline, 'comment'),
2567
2701
  (r'//.*$', Comment.Single)
@@ -2588,11 +2722,10 @@ class KokaLexer(RegexLexer):
2588
2722
  (r'[\'\n]', String.Char, '#pop'),
2589
2723
  ],
2590
2724
  'escape-sequence': [
2591
- (r'\\[abfnrtv0\\\"\'\?]', String.Escape),
2725
+ (r'\\[nrt\\\"\']', String.Escape),
2592
2726
  (r'\\x[0-9a-fA-F]{2}', String.Escape),
2593
2727
  (r'\\u[0-9a-fA-F]{4}', String.Escape),
2594
2728
  # Yes, \U literals are 6 hex digits.
2595
2729
  (r'\\U[0-9a-fA-F]{6}', String.Escape)
2596
2730
  ]
2597
2731
  }
2598
-