pygments.rb 0.5.4 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (267) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +40 -0
  3. data/README.md +3 -0
  4. data/lexers +0 -0
  5. data/lib/pygments/popen.rb +3 -2
  6. data/lib/pygments/version.rb +1 -1
  7. data/pygments.rb.gemspec +2 -1
  8. data/vendor/pygments-main/AUTHORS +20 -1
  9. data/vendor/pygments-main/CHANGES +55 -3
  10. data/vendor/pygments-main/LICENSE +1 -1
  11. data/vendor/pygments-main/MANIFEST.in +1 -1
  12. data/vendor/pygments-main/Makefile +5 -8
  13. data/vendor/pygments-main/REVISION +1 -1
  14. data/vendor/pygments-main/doc/Makefile +153 -0
  15. data/vendor/pygments-main/doc/_static/favicon.ico +0 -0
  16. data/vendor/pygments-main/doc/_static/logo_new.png +0 -0
  17. data/vendor/pygments-main/doc/_static/logo_only.png +0 -0
  18. data/vendor/pygments-main/doc/_templates/docssidebar.html +3 -0
  19. data/vendor/pygments-main/doc/_templates/indexsidebar.html +25 -0
  20. data/vendor/pygments-main/doc/_themes/pygments14/layout.html +98 -0
  21. data/vendor/pygments-main/doc/_themes/pygments14/static/bodybg.png +0 -0
  22. data/vendor/pygments-main/doc/_themes/pygments14/static/docbg.png +0 -0
  23. data/vendor/pygments-main/doc/_themes/pygments14/static/listitem.png +0 -0
  24. data/vendor/pygments-main/doc/_themes/pygments14/static/logo.png +0 -0
  25. data/vendor/pygments-main/doc/_themes/pygments14/static/pocoo.png +0 -0
  26. data/vendor/pygments-main/doc/_themes/pygments14/static/pygments14.css_t +401 -0
  27. data/vendor/pygments-main/doc/_themes/pygments14/theme.conf +15 -0
  28. data/vendor/pygments-main/doc/conf.py +249 -0
  29. data/vendor/pygments-main/doc/docs/api.rst +316 -0
  30. data/vendor/pygments-main/doc/docs/authors.rst +4 -0
  31. data/vendor/pygments-main/doc/docs/changelog.rst +1 -0
  32. data/vendor/pygments-main/{docs/src/cmdline.txt → doc/docs/cmdline.rst} +9 -11
  33. data/vendor/pygments-main/{docs/src/filterdevelopment.txt → doc/docs/filterdevelopment.rst} +1 -1
  34. data/vendor/pygments-main/{docs/src/filters.txt → doc/docs/filters.rst} +4 -5
  35. data/vendor/pygments-main/{docs/src/formatterdevelopment.txt → doc/docs/formatterdevelopment.rst} +1 -1
  36. data/vendor/pygments-main/{docs/src/formatters.txt → doc/docs/formatters.rst} +6 -6
  37. data/vendor/pygments-main/doc/docs/index.rst +66 -0
  38. data/vendor/pygments-main/{docs/src/integrate.txt → doc/docs/integrate.rst} +3 -7
  39. data/vendor/pygments-main/{docs/src/java.txt → doc/docs/java.rst} +0 -0
  40. data/vendor/pygments-main/{docs/src/lexerdevelopment.txt → doc/docs/lexerdevelopment.rst} +13 -14
  41. data/vendor/pygments-main/{docs/src/lexers.txt → doc/docs/lexers.rst} +7 -5
  42. data/vendor/pygments-main/{docs/src/moinmoin.txt → doc/docs/moinmoin.rst} +0 -0
  43. data/vendor/pygments-main/{docs/src/plugins.txt → doc/docs/plugins.rst} +0 -0
  44. data/vendor/pygments-main/{docs/src/quickstart.txt → doc/docs/quickstart.rst} +22 -19
  45. data/vendor/pygments-main/{docs/src/rstdirective.txt → doc/docs/rstdirective.rst} +0 -0
  46. data/vendor/pygments-main/{docs/src/styles.txt → doc/docs/styles.rst} +2 -2
  47. data/vendor/pygments-main/{docs/src/tokens.txt → doc/docs/tokens.rst} +10 -7
  48. data/vendor/pygments-main/{docs/src/unicode.txt → doc/docs/unicode.rst} +8 -7
  49. data/vendor/pygments-main/doc/download.rst +41 -0
  50. data/vendor/pygments-main/doc/faq.rst +143 -0
  51. data/vendor/pygments-main/doc/index.rst +53 -0
  52. data/vendor/pygments-main/doc/languages.rst +151 -0
  53. data/vendor/pygments-main/doc/make.bat +190 -0
  54. data/vendor/pygments-main/{docs → doc}/pygmentize.1 +0 -0
  55. data/vendor/pygments-main/external/autopygmentize +65 -48
  56. data/vendor/pygments-main/external/markdown-processor.py +15 -15
  57. data/vendor/pygments-main/external/moin-parser.py +1 -1
  58. data/vendor/pygments-main/external/rst-directive.py +2 -3
  59. data/vendor/pygments-main/ez_setup.py +340 -234
  60. data/vendor/pygments-main/pygments/__init__.py +4 -4
  61. data/vendor/pygments-main/pygments/cmdline.py +81 -68
  62. data/vendor/pygments-main/pygments/console.py +1 -1
  63. data/vendor/pygments-main/pygments/filter.py +1 -1
  64. data/vendor/pygments-main/pygments/filters/__init__.py +11 -9
  65. data/vendor/pygments-main/pygments/formatter.py +3 -3
  66. data/vendor/pygments-main/pygments/formatters/__init__.py +5 -3
  67. data/vendor/pygments-main/pygments/formatters/_mapping.py +16 -5
  68. data/vendor/pygments-main/pygments/formatters/bbcode.py +1 -1
  69. data/vendor/pygments-main/pygments/formatters/html.py +44 -26
  70. data/vendor/pygments-main/pygments/formatters/img.py +21 -14
  71. data/vendor/pygments-main/pygments/formatters/latex.py +104 -12
  72. data/vendor/pygments-main/pygments/formatters/other.py +53 -6
  73. data/vendor/pygments-main/pygments/formatters/rtf.py +30 -16
  74. data/vendor/pygments-main/pygments/formatters/svg.py +2 -2
  75. data/vendor/pygments-main/pygments/formatters/terminal.py +41 -1
  76. data/vendor/pygments-main/pygments/formatters/terminal256.py +2 -2
  77. data/vendor/pygments-main/pygments/lexer.py +59 -39
  78. data/vendor/pygments-main/pygments/lexers/__init__.py +38 -15
  79. data/vendor/pygments-main/pygments/lexers/_asybuiltins.py +1 -1
  80. data/vendor/pygments-main/pygments/lexers/_clbuiltins.py +15 -15
  81. data/vendor/pygments-main/pygments/lexers/_cocoabuiltins.py +73 -0
  82. data/vendor/pygments-main/pygments/lexers/_lassobuiltins.py +11 -1
  83. data/vendor/pygments-main/pygments/lexers/_luabuiltins.py +14 -8
  84. data/vendor/pygments-main/pygments/lexers/_mapping.py +81 -30
  85. data/vendor/pygments-main/pygments/lexers/_openedgebuiltins.py +1 -1
  86. data/vendor/pygments-main/pygments/lexers/_phpbuiltins.py +1067 -95
  87. data/vendor/pygments-main/pygments/lexers/_postgres_builtins.py +7 -7
  88. data/vendor/pygments-main/pygments/lexers/_robotframeworklexer.py +5 -4
  89. data/vendor/pygments-main/pygments/lexers/_scilab_builtins.py +1 -1
  90. data/vendor/pygments-main/pygments/lexers/_sourcemodbuiltins.py +12 -7
  91. data/vendor/pygments-main/pygments/lexers/_stan_builtins.py +112 -18
  92. data/vendor/pygments-main/pygments/lexers/_vimbuiltins.py +1 -1
  93. data/vendor/pygments-main/pygments/lexers/agile.py +644 -382
  94. data/vendor/pygments-main/pygments/lexers/asm.py +66 -29
  95. data/vendor/pygments-main/pygments/lexers/compiled.py +1643 -174
  96. data/vendor/pygments-main/pygments/lexers/dalvik.py +27 -6
  97. data/vendor/pygments-main/pygments/lexers/dotnet.py +45 -33
  98. data/vendor/pygments-main/pygments/lexers/foxpro.py +3 -3
  99. data/vendor/pygments-main/pygments/lexers/functional.py +1510 -570
  100. data/vendor/pygments-main/pygments/lexers/graph.py +81 -0
  101. data/vendor/pygments-main/pygments/lexers/hdl.py +29 -30
  102. data/vendor/pygments-main/pygments/lexers/inferno.py +96 -0
  103. data/vendor/pygments-main/pygments/lexers/jvm.py +622 -207
  104. data/vendor/pygments-main/pygments/lexers/math.py +438 -70
  105. data/vendor/pygments-main/pygments/lexers/other.py +885 -171
  106. data/vendor/pygments-main/pygments/lexers/parsers.py +24 -24
  107. data/vendor/pygments-main/pygments/lexers/qbasic.py +157 -0
  108. data/vendor/pygments-main/pygments/lexers/rdf.py +99 -0
  109. data/vendor/pygments-main/pygments/lexers/shell.py +28 -25
  110. data/vendor/pygments-main/pygments/lexers/special.py +7 -8
  111. data/vendor/pygments-main/pygments/lexers/sql.py +56 -23
  112. data/vendor/pygments-main/pygments/lexers/templates.py +377 -59
  113. data/vendor/pygments-main/pygments/lexers/text.py +208 -46
  114. data/vendor/pygments-main/pygments/lexers/web.py +729 -264
  115. data/vendor/pygments-main/pygments/modeline.py +1 -1
  116. data/vendor/pygments-main/pygments/plugin.py +1 -1
  117. data/vendor/pygments-main/pygments/scanner.py +1 -1
  118. data/vendor/pygments-main/pygments/sphinxext.py +153 -0
  119. data/vendor/pygments-main/pygments/style.py +3 -2
  120. data/vendor/pygments-main/pygments/styles/__init__.py +5 -1
  121. data/vendor/pygments-main/pygments/styles/autumn.py +1 -1
  122. data/vendor/pygments-main/pygments/styles/borland.py +1 -1
  123. data/vendor/pygments-main/pygments/styles/bw.py +1 -1
  124. data/vendor/pygments-main/pygments/styles/colorful.py +1 -1
  125. data/vendor/pygments-main/pygments/styles/default.py +1 -1
  126. data/vendor/pygments-main/pygments/styles/emacs.py +1 -1
  127. data/vendor/pygments-main/pygments/styles/friendly.py +1 -1
  128. data/vendor/pygments-main/pygments/styles/fruity.py +1 -1
  129. data/vendor/pygments-main/pygments/styles/igor.py +29 -0
  130. data/vendor/pygments-main/pygments/styles/manni.py +1 -1
  131. data/vendor/pygments-main/pygments/styles/monokai.py +4 -4
  132. data/vendor/pygments-main/pygments/styles/murphy.py +1 -1
  133. data/vendor/pygments-main/pygments/styles/native.py +1 -1
  134. data/vendor/pygments-main/pygments/styles/paraiso_dark.py +125 -0
  135. data/vendor/pygments-main/pygments/styles/paraiso_light.py +125 -0
  136. data/vendor/pygments-main/pygments/styles/pastie.py +1 -1
  137. data/vendor/pygments-main/pygments/styles/perldoc.py +1 -1
  138. data/vendor/pygments-main/pygments/styles/rrt.py +1 -1
  139. data/vendor/pygments-main/pygments/styles/tango.py +1 -1
  140. data/vendor/pygments-main/pygments/styles/trac.py +1 -1
  141. data/vendor/pygments-main/pygments/styles/vim.py +1 -1
  142. data/vendor/pygments-main/pygments/styles/vs.py +1 -1
  143. data/vendor/pygments-main/pygments/styles/xcode.py +51 -0
  144. data/vendor/pygments-main/pygments/token.py +4 -1
  145. data/vendor/pygments-main/pygments/unistring.py +6 -5
  146. data/vendor/pygments-main/pygments/util.py +35 -21
  147. data/vendor/pygments-main/scripts/check_sources.py +28 -44
  148. data/vendor/pygments-main/scripts/detect_missing_analyse_text.py +5 -4
  149. data/vendor/pygments-main/scripts/find_codetags.py +27 -19
  150. data/vendor/pygments-main/scripts/find_error.py +16 -13
  151. data/vendor/pygments-main/scripts/get_vimkw.py +3 -3
  152. data/vendor/pygments-main/scripts/vim2pygments.py +9 -7
  153. data/vendor/pygments-main/setup.py +19 -19
  154. data/vendor/pygments-main/tests/examplefiles/99_bottles_of_beer.chpl +118 -0
  155. data/vendor/pygments-main/tests/examplefiles/Error.pmod +38 -0
  156. data/vendor/pygments-main/tests/examplefiles/FakeFile.pike +360 -0
  157. data/vendor/pygments-main/tests/examplefiles/{ANTLRv3.g → antlr_ANTLRv3.g} +0 -0
  158. data/vendor/pygments-main/tests/examplefiles/clojure-weird-keywords.clj +5 -0
  159. data/vendor/pygments-main/tests/examplefiles/core.cljs +52 -0
  160. data/vendor/pygments-main/tests/examplefiles/demo.cfm +13 -1
  161. data/vendor/pygments-main/tests/examplefiles/demo.hbs +12 -0
  162. data/vendor/pygments-main/tests/examplefiles/ember.handlebars +33 -0
  163. data/vendor/pygments-main/tests/examplefiles/example.als +217 -0
  164. data/vendor/pygments-main/tests/examplefiles/example.c +1 -1
  165. data/vendor/pygments-main/tests/examplefiles/example.chai +6 -0
  166. data/vendor/pygments-main/tests/examplefiles/example.coffee +27 -0
  167. data/vendor/pygments-main/tests/examplefiles/example.e +124 -0
  168. data/vendor/pygments-main/tests/examplefiles/example.f90 +8 -0
  169. data/vendor/pygments-main/tests/examplefiles/example.feature +16 -0
  170. data/vendor/pygments-main/tests/examplefiles/example.gd +23 -0
  171. data/vendor/pygments-main/tests/examplefiles/example.gi +64 -0
  172. data/vendor/pygments-main/tests/examplefiles/example.groovy +2 -0
  173. data/vendor/pygments-main/tests/examplefiles/example.hs +27 -0
  174. data/vendor/pygments-main/tests/examplefiles/example.hx +44 -1
  175. data/vendor/pygments-main/tests/examplefiles/example.i6t +32 -0
  176. data/vendor/pygments-main/tests/examplefiles/example.i7x +45 -0
  177. data/vendor/pygments-main/tests/examplefiles/example.inf +374 -0
  178. data/vendor/pygments-main/tests/examplefiles/example.j +564 -0
  179. data/vendor/pygments-main/tests/examplefiles/example.java +16 -0
  180. data/vendor/pygments-main/tests/examplefiles/example.kal +75 -0
  181. data/vendor/pygments-main/tests/examplefiles/example.liquid +42 -0
  182. data/vendor/pygments-main/tests/examplefiles/example.ma +8 -0
  183. data/vendor/pygments-main/tests/examplefiles/example.mq4 +187 -0
  184. data/vendor/pygments-main/tests/examplefiles/example.mqh +123 -0
  185. data/vendor/pygments-main/tests/examplefiles/example.ni +57 -0
  186. data/vendor/pygments-main/tests/examplefiles/example.nix +80 -0
  187. data/vendor/pygments-main/tests/examplefiles/example.pp +8 -0
  188. data/vendor/pygments-main/tests/examplefiles/example.red +257 -0
  189. data/vendor/pygments-main/tests/examplefiles/example.reds +150 -0
  190. data/vendor/pygments-main/tests/examplefiles/example.rkt +677 -29
  191. data/vendor/pygments-main/tests/examplefiles/example.sh +22 -0
  192. data/vendor/pygments-main/tests/examplefiles/example.slim +31 -0
  193. data/vendor/pygments-main/tests/examplefiles/example.sls +51 -0
  194. data/vendor/pygments-main/tests/examplefiles/example.stan +2 -0
  195. data/vendor/pygments-main/tests/examplefiles/example.todotxt +9 -0
  196. data/vendor/pygments-main/tests/examplefiles/exampleScript.cfc +241 -0
  197. data/vendor/pygments-main/tests/examplefiles/exampleTag.cfc +18 -0
  198. data/vendor/pygments-main/tests/examplefiles/example_elixir.ex +192 -350
  199. data/vendor/pygments-main/tests/examplefiles/hash_syntax.rb +5 -0
  200. data/vendor/pygments-main/tests/examplefiles/hello.at +6 -0
  201. data/vendor/pygments-main/tests/examplefiles/hello.golo +5 -0
  202. data/vendor/pygments-main/tests/examplefiles/hello.lsl +12 -0
  203. data/vendor/pygments-main/tests/examplefiles/{File.hy → hybris_File.hy} +0 -0
  204. data/vendor/pygments-main/tests/examplefiles/{mg_sample.pro → idl_sample.pro} +0 -0
  205. data/vendor/pygments-main/tests/examplefiles/iex_example +23 -0
  206. data/vendor/pygments-main/tests/examplefiles/inet_pton6.dg +24 -24
  207. data/vendor/pygments-main/tests/examplefiles/language.hy +165 -0
  208. data/vendor/pygments-main/tests/examplefiles/limbo.b +456 -0
  209. data/vendor/pygments-main/tests/examplefiles/livescript-demo.ls +3 -1
  210. data/vendor/pygments-main/tests/examplefiles/main.cmake +2 -0
  211. data/vendor/pygments-main/tests/examplefiles/objc_example.m +163 -16
  212. data/vendor/pygments-main/tests/examplefiles/{example.p → openedge_example} +0 -0
  213. data/vendor/pygments-main/tests/examplefiles/pawn_example +25 -0
  214. data/vendor/pygments-main/tests/examplefiles/qbasic_example +2 -0
  215. data/vendor/pygments-main/tests/examplefiles/r6rs-comments.scm +23 -0
  216. data/vendor/pygments-main/tests/examplefiles/{robotframework.txt → robotframework_test.txt} +0 -0
  217. data/vendor/pygments-main/tests/examplefiles/rql-queries.rql +34 -0
  218. data/vendor/pygments-main/tests/examplefiles/scope.cirru +43 -0
  219. data/vendor/pygments-main/tests/examplefiles/sparql.rq +23 -0
  220. data/vendor/pygments-main/tests/examplefiles/test.R +37 -5
  221. data/vendor/pygments-main/tests/examplefiles/test.apl +26 -0
  222. data/vendor/pygments-main/tests/examplefiles/test.cyp +123 -0
  223. data/vendor/pygments-main/tests/examplefiles/test.idr +93 -0
  224. data/vendor/pygments-main/tests/examplefiles/test.mask +41 -0
  225. data/vendor/pygments-main/tests/examplefiles/test.pan +54 -0
  226. data/vendor/pygments-main/tests/examplefiles/test.php +3 -1
  227. data/vendor/pygments-main/tests/examplefiles/test.pig +148 -0
  228. data/vendor/pygments-main/tests/examplefiles/test.pwn +253 -0
  229. data/vendor/pygments-main/tests/examplefiles/test.r3 +27 -7
  230. data/vendor/pygments-main/tests/examplefiles/test.rsl +111 -0
  231. data/vendor/pygments-main/tests/examplefiles/test.swift +65 -0
  232. data/vendor/pygments-main/tests/examplefiles/test.zep +33 -0
  233. data/vendor/pygments-main/tests/examplefiles/unicode.go +10 -0
  234. data/vendor/pygments-main/tests/examplefiles/{test.bas → vbnet_test.bas} +0 -0
  235. data/vendor/pygments-main/tests/examplefiles/vctreestatus_hg +4 -0
  236. data/vendor/pygments-main/tests/examplefiles/vimrc +21 -0
  237. data/vendor/pygments-main/tests/run.py +14 -19
  238. data/vendor/pygments-main/tests/string_asserts.py +22 -0
  239. data/vendor/pygments-main/tests/test_basic_api.py +28 -14
  240. data/vendor/pygments-main/tests/test_cfm.py +46 -0
  241. data/vendor/pygments-main/tests/test_clexer.py +208 -3
  242. data/vendor/pygments-main/tests/test_cmdline.py +6 -5
  243. data/vendor/pygments-main/tests/test_examplefiles.py +30 -19
  244. data/vendor/pygments-main/tests/test_html_formatter.py +15 -13
  245. data/vendor/pygments-main/tests/test_latex_formatter.py +4 -2
  246. data/vendor/pygments-main/tests/test_lexers_other.py +11 -11
  247. data/vendor/pygments-main/tests/test_objectiveclexer.py +81 -0
  248. data/vendor/pygments-main/tests/test_perllexer.py +1 -1
  249. data/vendor/pygments-main/tests/test_qbasiclexer.py +43 -0
  250. data/vendor/pygments-main/tests/test_regexlexer.py +8 -1
  251. data/vendor/pygments-main/tests/test_rtf_formatter.py +109 -0
  252. data/vendor/pygments-main/tests/test_string_asserts.py +39 -0
  253. data/vendor/pygments-main/tests/test_token.py +3 -3
  254. data/vendor/pygments-main/tests/test_using_api.py +1 -1
  255. data/vendor/pygments-main/tests/test_util.py +1 -1
  256. metadata +138 -51
  257. data/vendor/pygments-main/docs/generate.py +0 -472
  258. data/vendor/pygments-main/docs/src/api.txt +0 -270
  259. data/vendor/pygments-main/docs/src/authors.txt +0 -5
  260. data/vendor/pygments-main/docs/src/changelog.txt +0 -5
  261. data/vendor/pygments-main/docs/src/index.txt +0 -69
  262. data/vendor/pygments-main/docs/src/installation.txt +0 -71
  263. data/vendor/pygments-main/external/rst-directive-old.py +0 -77
  264. data/vendor/pygments-main/scripts/reindent.py +0 -291
  265. data/vendor/pygments-main/tests/examplefiles/import.hs +0 -4
  266. data/vendor/pygments-main/tests/examplefiles/objc_example2.m +0 -24
  267. data/vendor/pygments-main/tests/old_run.py +0 -138
@@ -5,16 +5,16 @@
5
5
 
6
6
  Other formatters: NullFormatter, RawTokenFormatter.
7
7
 
8
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
8
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
9
9
  :license: BSD, see LICENSE for details.
10
10
  """
11
11
 
12
12
  from pygments.formatter import Formatter
13
- from pygments.util import OptionError, get_choice_opt, b
13
+ from pygments.util import OptionError, get_choice_opt
14
14
  from pygments.token import Token
15
15
  from pygments.console import colorize
16
16
 
17
- __all__ = ['NullFormatter', 'RawTokenFormatter']
17
+ __all__ = ['NullFormatter', 'RawTokenFormatter', 'TestcaseFormatter']
18
18
 
19
19
 
20
20
  class NullFormatter(Formatter):
@@ -40,7 +40,7 @@ class RawTokenFormatter(Formatter):
40
40
 
41
41
  The format is ``tokentype<TAB>repr(tokenstring)\n``. The output can later
42
42
  be converted to a token stream with the `RawTokenLexer`, described in the
43
- `lexer list <lexers.txt>`_.
43
+ :doc:`lexer list <lexers>`.
44
44
 
45
45
  Only two options are accepted:
46
46
 
@@ -50,7 +50,8 @@ class RawTokenFormatter(Formatter):
50
50
  `error_color`
51
51
  If set to a color name, highlight error tokens using that color. If
52
52
  set but with no value, defaults to ``'red'``.
53
- *New in Pygments 0.11.*
53
+
54
+ .. versionadded:: 0.11
54
55
 
55
56
  """
56
57
  name = 'Raw tokens'
@@ -79,7 +80,7 @@ class RawTokenFormatter(Formatter):
79
80
 
80
81
  def format(self, tokensource, outfile):
81
82
  try:
82
- outfile.write(b(''))
83
+ outfile.write(b'')
83
84
  except TypeError:
84
85
  raise TypeError('The raw tokens formatter needs a binary '
85
86
  'output file')
@@ -113,3 +114,49 @@ class RawTokenFormatter(Formatter):
113
114
  for ttype, value in tokensource:
114
115
  write("%s\t%r\n" % (ttype, value))
115
116
  flush()
117
+
118
+ TESTCASE_BEFORE = u'''\
119
+ def testNeedsName(self):
120
+ fragment = %r
121
+ tokens = [
122
+ '''
123
+ TESTCASE_AFTER = u'''\
124
+ ]
125
+ self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
126
+ '''
127
+
128
+
129
+ class TestcaseFormatter(Formatter):
130
+ """
131
+ Format tokens as appropriate for a new testcase.
132
+
133
+ .. versionadded:: 2.0
134
+ """
135
+ name = 'Testcase'
136
+ aliases = ['testcase']
137
+
138
+ def __init__(self, **options):
139
+ Formatter.__init__(self, **options)
140
+ #if self.encoding != 'utf-8':
141
+ # print >>sys.stderr, "NOTICE: Forcing encoding to utf-8, as all Pygments source is"
142
+ if self.encoding is not None and self.encoding != 'utf-8':
143
+ raise ValueError("Only None and utf-u are allowed encodings.")
144
+
145
+ def format(self, tokensource, outfile):
146
+ indentation = ' ' * 12
147
+ rawbuf = []
148
+ outbuf = []
149
+ for ttype, value in tokensource:
150
+ rawbuf.append(value)
151
+ outbuf.append('%s(%s, %r),\n' % (indentation, ttype, value))
152
+
153
+ before = TESTCASE_BEFORE % (u''.join(rawbuf),)
154
+ during = u''.join(outbuf)
155
+ after = TESTCASE_AFTER
156
+ if self.encoding is None:
157
+ outfile.write(before + during + after)
158
+ else:
159
+ outfile.write(before.encode('utf-8'))
160
+ outfile.write(during.encode('utf-8'))
161
+ outfile.write(after.encode('utf-8'))
162
+ outfile.flush()
@@ -5,11 +5,12 @@
5
5
 
6
6
  A formatter that generates RTF files.
7
7
 
8
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
8
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
9
9
  :license: BSD, see LICENSE for details.
10
10
  """
11
11
 
12
12
  from pygments.formatter import Formatter
13
+ from pygments.util import get_int_opt, _surrogatepair
13
14
 
14
15
 
15
16
  __all__ = ['RtfFormatter']
@@ -21,7 +22,11 @@ class RtfFormatter(Formatter):
21
22
  documents with color information and other useful stuff. Perfect for Copy and
22
23
  Paste into Microsoft® Word® documents.
23
24
 
24
- *New in Pygments 0.6.*
25
+ Please note that ``encoding`` and ``outencoding`` options are ignored.
26
+ The RTF format is ASCII natively, but handles unicode characters correctly
27
+ thanks to escape sequences.
28
+
29
+ .. versionadded:: 0.6
25
30
 
26
31
  Additional options accepted:
27
32
 
@@ -32,6 +37,12 @@ class RtfFormatter(Formatter):
32
37
  `fontface`
33
38
  The used font famliy, for example ``Bitstream Vera Sans``. Defaults to
34
39
  some generic font which is supposed to have fixed width.
40
+
41
+ `fontsize`
42
+ Size of the font used. Size is specified in half points. The
43
+ default is 24 half-points, giving a size 12 font.
44
+
45
+ .. versionadded:: 2.0
35
46
  """
36
47
  name = 'RTF'
37
48
  aliases = ['rtf']
@@ -49,9 +60,11 @@ class RtfFormatter(Formatter):
49
60
  specification claims that ``\fmodern`` are "Fixed-pitch serif
50
61
  and sans serif fonts". Hope every RTF implementation thinks
51
62
  the same about modern...
63
+
52
64
  """
53
65
  Formatter.__init__(self, **options)
54
66
  self.fontface = options.get('fontface') or ''
67
+ self.fontsize = get_int_opt(options, 'fontsize', 0)
55
68
 
56
69
  def _escape(self, text):
57
70
  return text.replace('\\', '\\\\') \
@@ -65,28 +78,27 @@ class RtfFormatter(Formatter):
65
78
 
66
79
  # escape text
67
80
  text = self._escape(text)
68
- if self.encoding in ('utf-8', 'utf-16', 'utf-32'):
69
- encoding = 'iso-8859-15'
70
- else:
71
- encoding = self.encoding or 'iso-8859-15'
72
81
 
73
82
  buf = []
74
83
  for c in text:
75
- if ord(c) > 128:
76
- ansic = c.encode(encoding, 'ignore') or '?'
77
- if ord(ansic) > 128:
78
- ansic = '\\\'%x' % ord(ansic)
79
- else:
80
- ansic = c
81
- buf.append(r'\ud{\u%d%s}' % (ord(c), ansic))
82
- else:
84
+ cn = ord(c)
85
+ if cn < (2**7):
86
+ # ASCII character
83
87
  buf.append(str(c))
88
+ elif (2**7) <= cn < (2**16):
89
+ # single unicode escape sequence
90
+ buf.append(r'{\u%d}' % cn)
91
+ elif (2**16) <= cn:
92
+ # RTF limits unicode to 16 bits.
93
+ # Force surrogate pairs
94
+ h,l = _surrogatepair(cn)
95
+ buf.append(r'{\u%d}{\u%d}' % (h,l))
84
96
 
85
97
  return ''.join(buf).replace('\n', '\\par\n')
86
98
 
87
99
  def format_unencoded(self, tokensource, outfile):
88
100
  # rtf 1.8 header
89
- outfile.write(r'{\rtf1\ansi\deff0'
101
+ outfile.write(r'{\rtf1\ansi\uc0\deff0'
90
102
  r'{\fonttbl{\f0\fmodern\fprq1\fcharset0%s;}}'
91
103
  r'{\colortbl;' % (self.fontface and
92
104
  ' ' + self._escape(self.fontface) or
@@ -105,7 +117,9 @@ class RtfFormatter(Formatter):
105
117
  int(color[4:6], 16)
106
118
  ))
107
119
  offset += 1
108
- outfile.write(r'}\f0')
120
+ outfile.write(r'}\f0 ')
121
+ if self.fontsize:
122
+ outfile.write(r'\fs%d' % (self.fontsize))
109
123
 
110
124
  # highlight stream
111
125
  for ttype, value in tokensource:
@@ -5,7 +5,7 @@
5
5
 
6
6
  Formatter for SVG output.
7
7
 
8
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
8
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
9
9
  :license: BSD, see LICENSE for details.
10
10
  """
11
11
 
@@ -35,7 +35,7 @@ class SvgFormatter(Formatter):
35
35
  By default, this formatter outputs a full SVG document including doctype
36
36
  declaration and the ``<svg>`` root element.
37
37
 
38
- *New in Pygments 0.9.*
38
+ .. versionadded:: 0.9
39
39
 
40
40
  Additional options accepted:
41
41
 
@@ -5,7 +5,7 @@
5
5
 
6
6
  Formatter for terminal output with ANSI sequences.
7
7
 
8
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
8
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
9
9
  :license: BSD, see LICENSE for details.
10
10
  """
11
11
 
@@ -73,6 +73,10 @@ class TerminalFormatter(Formatter):
73
73
  `colorscheme`
74
74
  A dictionary mapping token types to (lightbg, darkbg) color names or
75
75
  ``None`` (default: ``None`` = use builtin colorscheme).
76
+
77
+ `linenos`
78
+ Set to ``True`` to have line numbers on the terminal output as well
79
+ (default: ``False`` = no line numbers).
76
80
  """
77
81
  name = 'Terminal'
78
82
  aliases = ['terminal', 'console']
@@ -83,6 +87,8 @@ class TerminalFormatter(Formatter):
83
87
  self.darkbg = get_choice_opt(options, 'bg',
84
88
  ['light', 'dark'], 'light') == 'dark'
85
89
  self.colorscheme = options.get('colorscheme', None) or TERMINAL_COLORS
90
+ self.linenos = options.get('linenos', False)
91
+ self._lineno = 0
86
92
 
87
93
  def format(self, tokensource, outfile):
88
94
  # hack: if the output is a terminal and has an encoding set,
@@ -93,7 +99,40 @@ class TerminalFormatter(Formatter):
93
99
  self.encoding = outfile.encoding
94
100
  return Formatter.format(self, tokensource, outfile)
95
101
 
102
+ def _write_lineno(self, outfile):
103
+ self._lineno += 1
104
+ outfile.write("\n%04d: " % self._lineno)
105
+
106
+ def _format_unencoded_with_lineno(self, tokensource, outfile):
107
+ self._write_lineno(outfile)
108
+
109
+ for ttype, value in tokensource:
110
+ if value.endswith("\n"):
111
+ self._write_lineno(outfile)
112
+ value = value[:-1]
113
+ color = self.colorscheme.get(ttype)
114
+ while color is None:
115
+ ttype = ttype[:-1]
116
+ color = self.colorscheme.get(ttype)
117
+ if color:
118
+ color = color[self.darkbg]
119
+ spl = value.split('\n')
120
+ for line in spl[:-1]:
121
+ self._write_lineno(outfile)
122
+ if line:
123
+ outfile.write(ansiformat(color, line[:-1]))
124
+ if spl[-1]:
125
+ outfile.write(ansiformat(color, spl[-1]))
126
+ else:
127
+ outfile.write(value)
128
+
129
+ outfile.write("\n")
130
+
96
131
  def format_unencoded(self, tokensource, outfile):
132
+ if self.linenos:
133
+ self._format_unencoded_with_lineno(tokensource, outfile)
134
+ return
135
+
97
136
  for ttype, value in tokensource:
98
137
  color = self.colorscheme.get(ttype)
99
138
  while color is None:
@@ -110,3 +149,4 @@ class TerminalFormatter(Formatter):
110
149
  outfile.write(ansiformat(color, spl[-1]))
111
150
  else:
112
151
  outfile.write(value)
152
+
@@ -11,7 +11,7 @@
11
11
 
12
12
  Formatter version 1.
13
13
 
14
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
14
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
15
15
  :license: BSD, see LICENSE for details.
16
16
  """
17
17
 
@@ -76,7 +76,7 @@ class Terminal256Formatter(Formatter):
76
76
  and converts them to nearest ANSI 256-color escape sequences. Bold and
77
77
  underline attributes from the style are preserved (and displayed).
78
78
 
79
- *New in Pygments 0.9.*
79
+ .. versionadded:: 0.9
80
80
 
81
81
  Options accepted:
82
82
 
@@ -5,7 +5,7 @@
5
5
 
6
6
  Base lexer classes.
7
7
 
8
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
8
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
9
9
  :license: BSD, see LICENSE for details.
10
10
  """
11
11
  import re, itertools
@@ -14,18 +14,18 @@ from pygments.filter import apply_filters, Filter
14
14
  from pygments.filters import get_filter_by_name
15
15
  from pygments.token import Error, Text, Other, _TokenType
16
16
  from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
17
- make_analysator
17
+ make_analysator, text_type, add_metaclass, iteritems
18
18
 
19
19
 
20
20
  __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
21
- 'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this']
21
+ 'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this', 'default']
22
22
 
23
23
 
24
- _encoding_map = [('\xef\xbb\xbf', 'utf-8'),
25
- ('\xff\xfe\0\0', 'utf-32'),
26
- ('\0\0\xfe\xff', 'utf-32be'),
27
- ('\xff\xfe', 'utf-16'),
28
- ('\xfe\xff', 'utf-16be')]
24
+ _encoding_map = [(b'\xef\xbb\xbf', 'utf-8'),
25
+ (b'\xff\xfe\0\0', 'utf-32'),
26
+ (b'\0\0\xfe\xff', 'utf-32be'),
27
+ (b'\xff\xfe', 'utf-16'),
28
+ (b'\xfe\xff', 'utf-16be')]
29
29
 
30
30
  _default_analyse = staticmethod(lambda x: 0.0)
31
31
 
@@ -42,6 +42,7 @@ class LexerMeta(type):
42
42
  return type.__new__(cls, name, bases, d)
43
43
 
44
44
 
45
+ @add_metaclass(LexerMeta)
45
46
  class Lexer(object):
46
47
  """
47
48
  Lexer for a specific language.
@@ -55,7 +56,9 @@ class Lexer(object):
55
56
  ``ensurenl``
56
57
  Make sure that the input ends with a newline (default: True). This
57
58
  is required for some lexers that consume input linewise.
58
- *New in Pygments 1.3.*
59
+
60
+ .. versionadded:: 1.3
61
+
59
62
  ``tabsize``
60
63
  If given and greater than 0, expand tabs in the input (default: 0).
61
64
  ``encoding``
@@ -84,8 +87,6 @@ class Lexer(object):
84
87
  #: Priority, should multiple lexers match and no content is provided
85
88
  priority = 0
86
89
 
87
- __metaclass__ = LexerMeta
88
-
89
90
  def __init__(self, **options):
90
91
  self.options = options
91
92
  self.stripnl = get_bool_opt(options, 'stripnl', True)
@@ -136,7 +137,7 @@ class Lexer(object):
136
137
  Also preprocess the text, i.e. expand tabs and strip it if
137
138
  wanted and applies registered filters.
138
139
  """
139
- if not isinstance(text, unicode):
140
+ if not isinstance(text, text_type):
140
141
  if self.encoding == 'guess':
141
142
  try:
142
143
  text = text.decode('utf-8')
@@ -155,17 +156,18 @@ class Lexer(object):
155
156
  decoded = None
156
157
  for bom, encoding in _encoding_map:
157
158
  if text.startswith(bom):
158
- decoded = unicode(text[len(bom):], encoding,
159
- errors='replace')
159
+ decoded = text[len(bom):].decode(encoding, 'replace')
160
160
  break
161
161
  # no BOM found, so use chardet
162
162
  if decoded is None:
163
163
  enc = chardet.detect(text[:1024]) # Guess using first 1KB
164
- decoded = unicode(text, enc.get('encoding') or 'utf-8',
165
- errors='replace')
164
+ decoded = text.decode(enc.get('encoding') or 'utf-8',
165
+ 'replace')
166
166
  text = decoded
167
167
  else:
168
168
  text = text.decode(self.encoding)
169
+ if text.startswith(u'\ufeff'):
170
+ text = text[len(u'\ufeff'):]
169
171
  else:
170
172
  if text.startswith(u'\ufeff'):
171
173
  text = text[len(u'\ufeff'):]
@@ -192,7 +194,9 @@ class Lexer(object):
192
194
 
193
195
  def get_tokens_unprocessed(self, text):
194
196
  """
195
- Return an iterable of (tokentype, value) pairs.
197
+ Return an iterable of (index, tokentype, value) pairs where "index"
198
+ is the starting position of the token within the input text.
199
+
196
200
  In subclasses, implement this method as a generator to
197
201
  maximize effectiveness.
198
202
  """
@@ -379,6 +383,16 @@ def using(_other, **kwargs):
379
383
  return callback
380
384
 
381
385
 
386
+ class default:
387
+ """
388
+ Indicates a state or state action (e.g. #pop) to apply.
389
+ For example default('#pop') is equivalent to ('', Token, '#pop')
390
+ Note that state tuples may be used as well
391
+ """
392
+ def __init__(self, state):
393
+ self.state = state
394
+
395
+
382
396
  class RegexLexerMeta(LexerMeta):
383
397
  """
384
398
  Metaclass for RegexLexer, creates the self._tokens attribute from
@@ -448,12 +462,16 @@ class RegexLexerMeta(LexerMeta):
448
462
  if isinstance(tdef, _inherit):
449
463
  # processed already
450
464
  continue
465
+ if isinstance(tdef, default):
466
+ new_state = cls._process_new_state(tdef.state, unprocessed, processed)
467
+ tokens.append((re.compile('').match, None, new_state))
468
+ continue
451
469
 
452
470
  assert type(tdef) is tuple, "wrong rule def %r" % tdef
453
471
 
454
472
  try:
455
473
  rex = cls._process_regex(tdef[0], rflags)
456
- except Exception, err:
474
+ except Exception as err:
457
475
  raise ValueError("uncompilable regex %r in state %r of %r: %s" %
458
476
  (tdef[0], state, cls, err))
459
477
 
@@ -472,7 +490,7 @@ class RegexLexerMeta(LexerMeta):
472
490
  """Preprocess a dictionary of token definitions."""
473
491
  processed = cls._all_tokens[name] = {}
474
492
  tokendefs = tokendefs or cls.tokens[name]
475
- for state in tokendefs.keys():
493
+ for state in list(tokendefs):
476
494
  cls._process_state(tokendefs, processed, state)
477
495
  return processed
478
496
 
@@ -493,7 +511,7 @@ class RegexLexerMeta(LexerMeta):
493
511
  for c in itertools.chain((cls,), cls.__mro__):
494
512
  toks = c.__dict__.get('tokens', {})
495
513
 
496
- for state, items in toks.iteritems():
514
+ for state, items in iteritems(toks):
497
515
  curitems = tokens.get(state)
498
516
  if curitems is None:
499
517
  tokens[state] = items
@@ -533,13 +551,13 @@ class RegexLexerMeta(LexerMeta):
533
551
  return type.__call__(cls, *args, **kwds)
534
552
 
535
553
 
554
+ @add_metaclass(RegexLexerMeta)
536
555
  class RegexLexer(Lexer):
537
556
  """
538
557
  Base for simple stateful regular expression-based lexers.
539
558
  Simplifies the lexing process so that you need only
540
559
  provide a list of states and regular expressions.
541
560
  """
542
- __metaclass__ = RegexLexerMeta
543
561
 
544
562
  #: Flags for compiling the regular expressions.
545
563
  #: Defaults to MULTILINE.
@@ -578,11 +596,12 @@ class RegexLexer(Lexer):
578
596
  for rexmatch, action, new_state in statetokens:
579
597
  m = rexmatch(text, pos)
580
598
  if m:
581
- if type(action) is _TokenType:
582
- yield pos, action, m.group()
583
- else:
584
- for item in action(self, m):
585
- yield item
599
+ if action is not None:
600
+ if type(action) is _TokenType:
601
+ yield pos, action, m.group()
602
+ else:
603
+ for item in action(self, m):
604
+ yield item
586
605
  pos = m.end()
587
606
  if new_state is not None:
588
607
  # state transition
@@ -656,15 +675,16 @@ class ExtendedRegexLexer(RegexLexer):
656
675
  for rexmatch, action, new_state in statetokens:
657
676
  m = rexmatch(text, ctx.pos, ctx.end)
658
677
  if m:
659
- if type(action) is _TokenType:
660
- yield ctx.pos, action, m.group()
661
- ctx.pos = m.end()
662
- else:
663
- for item in action(self, m, ctx):
664
- yield item
665
- if not new_state:
666
- # altered the state stack?
667
- statetokens = tokendefs[ctx.stack[-1]]
678
+ if action is not None:
679
+ if type(action) is _TokenType:
680
+ yield ctx.pos, action, m.group()
681
+ ctx.pos = m.end()
682
+ else:
683
+ for item in action(self, m, ctx):
684
+ yield item
685
+ if not new_state:
686
+ # altered the state stack?
687
+ statetokens = tokendefs[ctx.stack[-1]]
668
688
  # CAUTION: callback must set ctx.pos!
669
689
  if new_state is not None:
670
690
  # state transition
@@ -673,7 +693,7 @@ class ExtendedRegexLexer(RegexLexer):
673
693
  if state == '#pop':
674
694
  ctx.stack.pop()
675
695
  elif state == '#push':
676
- ctx.stack.append(statestack[-1])
696
+ ctx.stack.append(ctx.stack[-1])
677
697
  else:
678
698
  ctx.stack.append(state)
679
699
  elif isinstance(new_state, int):
@@ -718,7 +738,7 @@ def do_insertions(insertions, tokens):
718
738
  """
719
739
  insertions = iter(insertions)
720
740
  try:
721
- index, itokens = insertions.next()
741
+ index, itokens = next(insertions)
722
742
  except StopIteration:
723
743
  # no insertions
724
744
  for item in tokens:
@@ -744,7 +764,7 @@ def do_insertions(insertions, tokens):
744
764
  realpos += len(it_value)
745
765
  oldi = index - i
746
766
  try:
747
- index, itokens = insertions.next()
767
+ index, itokens = next(insertions)
748
768
  except StopIteration:
749
769
  insleft = False
750
770
  break # not strictly necessary
@@ -759,7 +779,7 @@ def do_insertions(insertions, tokens):
759
779
  yield realpos, t, v
760
780
  realpos += len(v)
761
781
  try:
762
- index, itokens = insertions.next()
782
+ index, itokens = next(insertions)
763
783
  except StopIteration:
764
784
  insleft = False
765
785
  break # not strictly necessary