pygments.rb 0.5.4 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (267) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +40 -0
  3. data/README.md +3 -0
  4. data/lexers +0 -0
  5. data/lib/pygments/popen.rb +3 -2
  6. data/lib/pygments/version.rb +1 -1
  7. data/pygments.rb.gemspec +2 -1
  8. data/vendor/pygments-main/AUTHORS +20 -1
  9. data/vendor/pygments-main/CHANGES +55 -3
  10. data/vendor/pygments-main/LICENSE +1 -1
  11. data/vendor/pygments-main/MANIFEST.in +1 -1
  12. data/vendor/pygments-main/Makefile +5 -8
  13. data/vendor/pygments-main/REVISION +1 -1
  14. data/vendor/pygments-main/doc/Makefile +153 -0
  15. data/vendor/pygments-main/doc/_static/favicon.ico +0 -0
  16. data/vendor/pygments-main/doc/_static/logo_new.png +0 -0
  17. data/vendor/pygments-main/doc/_static/logo_only.png +0 -0
  18. data/vendor/pygments-main/doc/_templates/docssidebar.html +3 -0
  19. data/vendor/pygments-main/doc/_templates/indexsidebar.html +25 -0
  20. data/vendor/pygments-main/doc/_themes/pygments14/layout.html +98 -0
  21. data/vendor/pygments-main/doc/_themes/pygments14/static/bodybg.png +0 -0
  22. data/vendor/pygments-main/doc/_themes/pygments14/static/docbg.png +0 -0
  23. data/vendor/pygments-main/doc/_themes/pygments14/static/listitem.png +0 -0
  24. data/vendor/pygments-main/doc/_themes/pygments14/static/logo.png +0 -0
  25. data/vendor/pygments-main/doc/_themes/pygments14/static/pocoo.png +0 -0
  26. data/vendor/pygments-main/doc/_themes/pygments14/static/pygments14.css_t +401 -0
  27. data/vendor/pygments-main/doc/_themes/pygments14/theme.conf +15 -0
  28. data/vendor/pygments-main/doc/conf.py +249 -0
  29. data/vendor/pygments-main/doc/docs/api.rst +316 -0
  30. data/vendor/pygments-main/doc/docs/authors.rst +4 -0
  31. data/vendor/pygments-main/doc/docs/changelog.rst +1 -0
  32. data/vendor/pygments-main/{docs/src/cmdline.txt → doc/docs/cmdline.rst} +9 -11
  33. data/vendor/pygments-main/{docs/src/filterdevelopment.txt → doc/docs/filterdevelopment.rst} +1 -1
  34. data/vendor/pygments-main/{docs/src/filters.txt → doc/docs/filters.rst} +4 -5
  35. data/vendor/pygments-main/{docs/src/formatterdevelopment.txt → doc/docs/formatterdevelopment.rst} +1 -1
  36. data/vendor/pygments-main/{docs/src/formatters.txt → doc/docs/formatters.rst} +6 -6
  37. data/vendor/pygments-main/doc/docs/index.rst +66 -0
  38. data/vendor/pygments-main/{docs/src/integrate.txt → doc/docs/integrate.rst} +3 -7
  39. data/vendor/pygments-main/{docs/src/java.txt → doc/docs/java.rst} +0 -0
  40. data/vendor/pygments-main/{docs/src/lexerdevelopment.txt → doc/docs/lexerdevelopment.rst} +13 -14
  41. data/vendor/pygments-main/{docs/src/lexers.txt → doc/docs/lexers.rst} +7 -5
  42. data/vendor/pygments-main/{docs/src/moinmoin.txt → doc/docs/moinmoin.rst} +0 -0
  43. data/vendor/pygments-main/{docs/src/plugins.txt → doc/docs/plugins.rst} +0 -0
  44. data/vendor/pygments-main/{docs/src/quickstart.txt → doc/docs/quickstart.rst} +22 -19
  45. data/vendor/pygments-main/{docs/src/rstdirective.txt → doc/docs/rstdirective.rst} +0 -0
  46. data/vendor/pygments-main/{docs/src/styles.txt → doc/docs/styles.rst} +2 -2
  47. data/vendor/pygments-main/{docs/src/tokens.txt → doc/docs/tokens.rst} +10 -7
  48. data/vendor/pygments-main/{docs/src/unicode.txt → doc/docs/unicode.rst} +8 -7
  49. data/vendor/pygments-main/doc/download.rst +41 -0
  50. data/vendor/pygments-main/doc/faq.rst +143 -0
  51. data/vendor/pygments-main/doc/index.rst +53 -0
  52. data/vendor/pygments-main/doc/languages.rst +151 -0
  53. data/vendor/pygments-main/doc/make.bat +190 -0
  54. data/vendor/pygments-main/{docs → doc}/pygmentize.1 +0 -0
  55. data/vendor/pygments-main/external/autopygmentize +65 -48
  56. data/vendor/pygments-main/external/markdown-processor.py +15 -15
  57. data/vendor/pygments-main/external/moin-parser.py +1 -1
  58. data/vendor/pygments-main/external/rst-directive.py +2 -3
  59. data/vendor/pygments-main/ez_setup.py +340 -234
  60. data/vendor/pygments-main/pygments/__init__.py +4 -4
  61. data/vendor/pygments-main/pygments/cmdline.py +81 -68
  62. data/vendor/pygments-main/pygments/console.py +1 -1
  63. data/vendor/pygments-main/pygments/filter.py +1 -1
  64. data/vendor/pygments-main/pygments/filters/__init__.py +11 -9
  65. data/vendor/pygments-main/pygments/formatter.py +3 -3
  66. data/vendor/pygments-main/pygments/formatters/__init__.py +5 -3
  67. data/vendor/pygments-main/pygments/formatters/_mapping.py +16 -5
  68. data/vendor/pygments-main/pygments/formatters/bbcode.py +1 -1
  69. data/vendor/pygments-main/pygments/formatters/html.py +44 -26
  70. data/vendor/pygments-main/pygments/formatters/img.py +21 -14
  71. data/vendor/pygments-main/pygments/formatters/latex.py +104 -12
  72. data/vendor/pygments-main/pygments/formatters/other.py +53 -6
  73. data/vendor/pygments-main/pygments/formatters/rtf.py +30 -16
  74. data/vendor/pygments-main/pygments/formatters/svg.py +2 -2
  75. data/vendor/pygments-main/pygments/formatters/terminal.py +41 -1
  76. data/vendor/pygments-main/pygments/formatters/terminal256.py +2 -2
  77. data/vendor/pygments-main/pygments/lexer.py +59 -39
  78. data/vendor/pygments-main/pygments/lexers/__init__.py +38 -15
  79. data/vendor/pygments-main/pygments/lexers/_asybuiltins.py +1 -1
  80. data/vendor/pygments-main/pygments/lexers/_clbuiltins.py +15 -15
  81. data/vendor/pygments-main/pygments/lexers/_cocoabuiltins.py +73 -0
  82. data/vendor/pygments-main/pygments/lexers/_lassobuiltins.py +11 -1
  83. data/vendor/pygments-main/pygments/lexers/_luabuiltins.py +14 -8
  84. data/vendor/pygments-main/pygments/lexers/_mapping.py +81 -30
  85. data/vendor/pygments-main/pygments/lexers/_openedgebuiltins.py +1 -1
  86. data/vendor/pygments-main/pygments/lexers/_phpbuiltins.py +1067 -95
  87. data/vendor/pygments-main/pygments/lexers/_postgres_builtins.py +7 -7
  88. data/vendor/pygments-main/pygments/lexers/_robotframeworklexer.py +5 -4
  89. data/vendor/pygments-main/pygments/lexers/_scilab_builtins.py +1 -1
  90. data/vendor/pygments-main/pygments/lexers/_sourcemodbuiltins.py +12 -7
  91. data/vendor/pygments-main/pygments/lexers/_stan_builtins.py +112 -18
  92. data/vendor/pygments-main/pygments/lexers/_vimbuiltins.py +1 -1
  93. data/vendor/pygments-main/pygments/lexers/agile.py +644 -382
  94. data/vendor/pygments-main/pygments/lexers/asm.py +66 -29
  95. data/vendor/pygments-main/pygments/lexers/compiled.py +1643 -174
  96. data/vendor/pygments-main/pygments/lexers/dalvik.py +27 -6
  97. data/vendor/pygments-main/pygments/lexers/dotnet.py +45 -33
  98. data/vendor/pygments-main/pygments/lexers/foxpro.py +3 -3
  99. data/vendor/pygments-main/pygments/lexers/functional.py +1510 -570
  100. data/vendor/pygments-main/pygments/lexers/graph.py +81 -0
  101. data/vendor/pygments-main/pygments/lexers/hdl.py +29 -30
  102. data/vendor/pygments-main/pygments/lexers/inferno.py +96 -0
  103. data/vendor/pygments-main/pygments/lexers/jvm.py +622 -207
  104. data/vendor/pygments-main/pygments/lexers/math.py +438 -70
  105. data/vendor/pygments-main/pygments/lexers/other.py +885 -171
  106. data/vendor/pygments-main/pygments/lexers/parsers.py +24 -24
  107. data/vendor/pygments-main/pygments/lexers/qbasic.py +157 -0
  108. data/vendor/pygments-main/pygments/lexers/rdf.py +99 -0
  109. data/vendor/pygments-main/pygments/lexers/shell.py +28 -25
  110. data/vendor/pygments-main/pygments/lexers/special.py +7 -8
  111. data/vendor/pygments-main/pygments/lexers/sql.py +56 -23
  112. data/vendor/pygments-main/pygments/lexers/templates.py +377 -59
  113. data/vendor/pygments-main/pygments/lexers/text.py +208 -46
  114. data/vendor/pygments-main/pygments/lexers/web.py +729 -264
  115. data/vendor/pygments-main/pygments/modeline.py +1 -1
  116. data/vendor/pygments-main/pygments/plugin.py +1 -1
  117. data/vendor/pygments-main/pygments/scanner.py +1 -1
  118. data/vendor/pygments-main/pygments/sphinxext.py +153 -0
  119. data/vendor/pygments-main/pygments/style.py +3 -2
  120. data/vendor/pygments-main/pygments/styles/__init__.py +5 -1
  121. data/vendor/pygments-main/pygments/styles/autumn.py +1 -1
  122. data/vendor/pygments-main/pygments/styles/borland.py +1 -1
  123. data/vendor/pygments-main/pygments/styles/bw.py +1 -1
  124. data/vendor/pygments-main/pygments/styles/colorful.py +1 -1
  125. data/vendor/pygments-main/pygments/styles/default.py +1 -1
  126. data/vendor/pygments-main/pygments/styles/emacs.py +1 -1
  127. data/vendor/pygments-main/pygments/styles/friendly.py +1 -1
  128. data/vendor/pygments-main/pygments/styles/fruity.py +1 -1
  129. data/vendor/pygments-main/pygments/styles/igor.py +29 -0
  130. data/vendor/pygments-main/pygments/styles/manni.py +1 -1
  131. data/vendor/pygments-main/pygments/styles/monokai.py +4 -4
  132. data/vendor/pygments-main/pygments/styles/murphy.py +1 -1
  133. data/vendor/pygments-main/pygments/styles/native.py +1 -1
  134. data/vendor/pygments-main/pygments/styles/paraiso_dark.py +125 -0
  135. data/vendor/pygments-main/pygments/styles/paraiso_light.py +125 -0
  136. data/vendor/pygments-main/pygments/styles/pastie.py +1 -1
  137. data/vendor/pygments-main/pygments/styles/perldoc.py +1 -1
  138. data/vendor/pygments-main/pygments/styles/rrt.py +1 -1
  139. data/vendor/pygments-main/pygments/styles/tango.py +1 -1
  140. data/vendor/pygments-main/pygments/styles/trac.py +1 -1
  141. data/vendor/pygments-main/pygments/styles/vim.py +1 -1
  142. data/vendor/pygments-main/pygments/styles/vs.py +1 -1
  143. data/vendor/pygments-main/pygments/styles/xcode.py +51 -0
  144. data/vendor/pygments-main/pygments/token.py +4 -1
  145. data/vendor/pygments-main/pygments/unistring.py +6 -5
  146. data/vendor/pygments-main/pygments/util.py +35 -21
  147. data/vendor/pygments-main/scripts/check_sources.py +28 -44
  148. data/vendor/pygments-main/scripts/detect_missing_analyse_text.py +5 -4
  149. data/vendor/pygments-main/scripts/find_codetags.py +27 -19
  150. data/vendor/pygments-main/scripts/find_error.py +16 -13
  151. data/vendor/pygments-main/scripts/get_vimkw.py +3 -3
  152. data/vendor/pygments-main/scripts/vim2pygments.py +9 -7
  153. data/vendor/pygments-main/setup.py +19 -19
  154. data/vendor/pygments-main/tests/examplefiles/99_bottles_of_beer.chpl +118 -0
  155. data/vendor/pygments-main/tests/examplefiles/Error.pmod +38 -0
  156. data/vendor/pygments-main/tests/examplefiles/FakeFile.pike +360 -0
  157. data/vendor/pygments-main/tests/examplefiles/{ANTLRv3.g → antlr_ANTLRv3.g} +0 -0
  158. data/vendor/pygments-main/tests/examplefiles/clojure-weird-keywords.clj +5 -0
  159. data/vendor/pygments-main/tests/examplefiles/core.cljs +52 -0
  160. data/vendor/pygments-main/tests/examplefiles/demo.cfm +13 -1
  161. data/vendor/pygments-main/tests/examplefiles/demo.hbs +12 -0
  162. data/vendor/pygments-main/tests/examplefiles/ember.handlebars +33 -0
  163. data/vendor/pygments-main/tests/examplefiles/example.als +217 -0
  164. data/vendor/pygments-main/tests/examplefiles/example.c +1 -1
  165. data/vendor/pygments-main/tests/examplefiles/example.chai +6 -0
  166. data/vendor/pygments-main/tests/examplefiles/example.coffee +27 -0
  167. data/vendor/pygments-main/tests/examplefiles/example.e +124 -0
  168. data/vendor/pygments-main/tests/examplefiles/example.f90 +8 -0
  169. data/vendor/pygments-main/tests/examplefiles/example.feature +16 -0
  170. data/vendor/pygments-main/tests/examplefiles/example.gd +23 -0
  171. data/vendor/pygments-main/tests/examplefiles/example.gi +64 -0
  172. data/vendor/pygments-main/tests/examplefiles/example.groovy +2 -0
  173. data/vendor/pygments-main/tests/examplefiles/example.hs +27 -0
  174. data/vendor/pygments-main/tests/examplefiles/example.hx +44 -1
  175. data/vendor/pygments-main/tests/examplefiles/example.i6t +32 -0
  176. data/vendor/pygments-main/tests/examplefiles/example.i7x +45 -0
  177. data/vendor/pygments-main/tests/examplefiles/example.inf +374 -0
  178. data/vendor/pygments-main/tests/examplefiles/example.j +564 -0
  179. data/vendor/pygments-main/tests/examplefiles/example.java +16 -0
  180. data/vendor/pygments-main/tests/examplefiles/example.kal +75 -0
  181. data/vendor/pygments-main/tests/examplefiles/example.liquid +42 -0
  182. data/vendor/pygments-main/tests/examplefiles/example.ma +8 -0
  183. data/vendor/pygments-main/tests/examplefiles/example.mq4 +187 -0
  184. data/vendor/pygments-main/tests/examplefiles/example.mqh +123 -0
  185. data/vendor/pygments-main/tests/examplefiles/example.ni +57 -0
  186. data/vendor/pygments-main/tests/examplefiles/example.nix +80 -0
  187. data/vendor/pygments-main/tests/examplefiles/example.pp +8 -0
  188. data/vendor/pygments-main/tests/examplefiles/example.red +257 -0
  189. data/vendor/pygments-main/tests/examplefiles/example.reds +150 -0
  190. data/vendor/pygments-main/tests/examplefiles/example.rkt +677 -29
  191. data/vendor/pygments-main/tests/examplefiles/example.sh +22 -0
  192. data/vendor/pygments-main/tests/examplefiles/example.slim +31 -0
  193. data/vendor/pygments-main/tests/examplefiles/example.sls +51 -0
  194. data/vendor/pygments-main/tests/examplefiles/example.stan +2 -0
  195. data/vendor/pygments-main/tests/examplefiles/example.todotxt +9 -0
  196. data/vendor/pygments-main/tests/examplefiles/exampleScript.cfc +241 -0
  197. data/vendor/pygments-main/tests/examplefiles/exampleTag.cfc +18 -0
  198. data/vendor/pygments-main/tests/examplefiles/example_elixir.ex +192 -350
  199. data/vendor/pygments-main/tests/examplefiles/hash_syntax.rb +5 -0
  200. data/vendor/pygments-main/tests/examplefiles/hello.at +6 -0
  201. data/vendor/pygments-main/tests/examplefiles/hello.golo +5 -0
  202. data/vendor/pygments-main/tests/examplefiles/hello.lsl +12 -0
  203. data/vendor/pygments-main/tests/examplefiles/{File.hy → hybris_File.hy} +0 -0
  204. data/vendor/pygments-main/tests/examplefiles/{mg_sample.pro → idl_sample.pro} +0 -0
  205. data/vendor/pygments-main/tests/examplefiles/iex_example +23 -0
  206. data/vendor/pygments-main/tests/examplefiles/inet_pton6.dg +24 -24
  207. data/vendor/pygments-main/tests/examplefiles/language.hy +165 -0
  208. data/vendor/pygments-main/tests/examplefiles/limbo.b +456 -0
  209. data/vendor/pygments-main/tests/examplefiles/livescript-demo.ls +3 -1
  210. data/vendor/pygments-main/tests/examplefiles/main.cmake +2 -0
  211. data/vendor/pygments-main/tests/examplefiles/objc_example.m +163 -16
  212. data/vendor/pygments-main/tests/examplefiles/{example.p → openedge_example} +0 -0
  213. data/vendor/pygments-main/tests/examplefiles/pawn_example +25 -0
  214. data/vendor/pygments-main/tests/examplefiles/qbasic_example +2 -0
  215. data/vendor/pygments-main/tests/examplefiles/r6rs-comments.scm +23 -0
  216. data/vendor/pygments-main/tests/examplefiles/{robotframework.txt → robotframework_test.txt} +0 -0
  217. data/vendor/pygments-main/tests/examplefiles/rql-queries.rql +34 -0
  218. data/vendor/pygments-main/tests/examplefiles/scope.cirru +43 -0
  219. data/vendor/pygments-main/tests/examplefiles/sparql.rq +23 -0
  220. data/vendor/pygments-main/tests/examplefiles/test.R +37 -5
  221. data/vendor/pygments-main/tests/examplefiles/test.apl +26 -0
  222. data/vendor/pygments-main/tests/examplefiles/test.cyp +123 -0
  223. data/vendor/pygments-main/tests/examplefiles/test.idr +93 -0
  224. data/vendor/pygments-main/tests/examplefiles/test.mask +41 -0
  225. data/vendor/pygments-main/tests/examplefiles/test.pan +54 -0
  226. data/vendor/pygments-main/tests/examplefiles/test.php +3 -1
  227. data/vendor/pygments-main/tests/examplefiles/test.pig +148 -0
  228. data/vendor/pygments-main/tests/examplefiles/test.pwn +253 -0
  229. data/vendor/pygments-main/tests/examplefiles/test.r3 +27 -7
  230. data/vendor/pygments-main/tests/examplefiles/test.rsl +111 -0
  231. data/vendor/pygments-main/tests/examplefiles/test.swift +65 -0
  232. data/vendor/pygments-main/tests/examplefiles/test.zep +33 -0
  233. data/vendor/pygments-main/tests/examplefiles/unicode.go +10 -0
  234. data/vendor/pygments-main/tests/examplefiles/{test.bas → vbnet_test.bas} +0 -0
  235. data/vendor/pygments-main/tests/examplefiles/vctreestatus_hg +4 -0
  236. data/vendor/pygments-main/tests/examplefiles/vimrc +21 -0
  237. data/vendor/pygments-main/tests/run.py +14 -19
  238. data/vendor/pygments-main/tests/string_asserts.py +22 -0
  239. data/vendor/pygments-main/tests/test_basic_api.py +28 -14
  240. data/vendor/pygments-main/tests/test_cfm.py +46 -0
  241. data/vendor/pygments-main/tests/test_clexer.py +208 -3
  242. data/vendor/pygments-main/tests/test_cmdline.py +6 -5
  243. data/vendor/pygments-main/tests/test_examplefiles.py +30 -19
  244. data/vendor/pygments-main/tests/test_html_formatter.py +15 -13
  245. data/vendor/pygments-main/tests/test_latex_formatter.py +4 -2
  246. data/vendor/pygments-main/tests/test_lexers_other.py +11 -11
  247. data/vendor/pygments-main/tests/test_objectiveclexer.py +81 -0
  248. data/vendor/pygments-main/tests/test_perllexer.py +1 -1
  249. data/vendor/pygments-main/tests/test_qbasiclexer.py +43 -0
  250. data/vendor/pygments-main/tests/test_regexlexer.py +8 -1
  251. data/vendor/pygments-main/tests/test_rtf_formatter.py +109 -0
  252. data/vendor/pygments-main/tests/test_string_asserts.py +39 -0
  253. data/vendor/pygments-main/tests/test_token.py +3 -3
  254. data/vendor/pygments-main/tests/test_using_api.py +1 -1
  255. data/vendor/pygments-main/tests/test_util.py +1 -1
  256. metadata +138 -51
  257. data/vendor/pygments-main/docs/generate.py +0 -472
  258. data/vendor/pygments-main/docs/src/api.txt +0 -270
  259. data/vendor/pygments-main/docs/src/authors.txt +0 -5
  260. data/vendor/pygments-main/docs/src/changelog.txt +0 -5
  261. data/vendor/pygments-main/docs/src/index.txt +0 -69
  262. data/vendor/pygments-main/docs/src/installation.txt +0 -71
  263. data/vendor/pygments-main/external/rst-directive-old.py +0 -77
  264. data/vendor/pygments-main/scripts/reindent.py +0 -291
  265. data/vendor/pygments-main/tests/examplefiles/import.hs +0 -4
  266. data/vendor/pygments-main/tests/examplefiles/objc_example2.m +0 -24
  267. data/vendor/pygments-main/tests/old_run.py +0 -138
@@ -3,17 +3,18 @@
3
3
  Command line test
4
4
  ~~~~~~~~~~~~~~~~~
5
5
 
6
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
6
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
7
7
  :license: BSD, see LICENSE for details.
8
8
  """
9
9
 
10
10
  # Test the command line interface
11
11
 
12
- import sys, os
12
+ import io
13
+ import sys
13
14
  import unittest
14
- import StringIO
15
15
 
16
16
  from pygments import highlight
17
+ from pygments.util import StringIO
17
18
  from pygments.cmdline import main as cmdline_main
18
19
 
19
20
  import support
@@ -24,8 +25,8 @@ TESTFILE, TESTDIR = support.location(__file__)
24
25
  def run_cmdline(*args):
25
26
  saved_stdout = sys.stdout
26
27
  saved_stderr = sys.stderr
27
- new_stdout = sys.stdout = StringIO.StringIO()
28
- new_stderr = sys.stderr = StringIO.StringIO()
28
+ new_stdout = sys.stdout = StringIO()
29
+ new_stderr = sys.stderr = StringIO()
29
30
  try:
30
31
  ret = cmdline_main(["pygmentize"] + list(args))
31
32
  finally:
@@ -3,18 +3,20 @@
3
3
  Pygments tests with example files
4
4
  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5
5
 
6
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
6
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
7
7
  :license: BSD, see LICENSE for details.
8
8
  """
9
9
 
10
+ from __future__ import print_function
11
+
10
12
  import os
11
13
  import pprint
12
14
  import difflib
13
- import cPickle as pickle
15
+ import pickle
14
16
 
15
17
  from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
16
18
  from pygments.token import Error
17
- from pygments.util import ClassNotFound, b
19
+ from pygments.util import ClassNotFound
18
20
 
19
21
  STORE_OUTPUT = False
20
22
 
@@ -31,21 +33,30 @@ def test_example_files():
31
33
  absfn = os.path.join(testdir, 'examplefiles', fn)
32
34
  if not os.path.isfile(absfn):
33
35
  continue
34
- outfn = os.path.join(outdir, fn)
35
36
 
37
+ print(absfn)
38
+ code = open(absfn, 'rb').read()
36
39
  try:
37
- lx = get_lexer_for_filename(absfn)
38
- except ClassNotFound:
39
- if "_" not in fn:
40
+ code = code.decode('utf-8')
41
+ except UnicodeError:
42
+ code = code.decode('latin1')
43
+
44
+ outfn = os.path.join(outdir, fn)
45
+
46
+ lx = None
47
+ if '_' in fn:
48
+ try:
49
+ lx = get_lexer_by_name(fn.split('_')[0])
50
+ except ClassNotFound:
51
+ pass
52
+ if lx is None:
53
+ try:
54
+ lx = get_lexer_for_filename(absfn, code=code)
55
+ except ClassNotFound:
40
56
  raise AssertionError('file %r has no registered extension, '
41
57
  'nor is of the form <lexer>_filename '
42
58
  'for overriding, thus no lexer found.'
43
- % fn)
44
- try:
45
- name, rest = fn.split("_", 1)
46
- lx = get_lexer_by_name(name)
47
- except ClassNotFound:
48
- raise AssertionError('no lexer found for file %r' % fn)
59
+ % fn)
49
60
  yield check_lexer, lx, absfn, outfn
50
61
 
51
62
  def check_lexer(lx, absfn, outfn):
@@ -54,8 +65,8 @@ def check_lexer(lx, absfn, outfn):
54
65
  text = fp.read()
55
66
  finally:
56
67
  fp.close()
57
- text = text.replace(b('\r\n'), b('\n'))
58
- text = text.strip(b('\n')) + b('\n')
68
+ text = text.replace(b'\r\n', b'\n')
69
+ text = text.strip(b'\n') + b'\n'
59
70
  try:
60
71
  text = text.decode('utf-8')
61
72
  if text.startswith(u'\ufeff'):
@@ -71,8 +82,8 @@ def check_lexer(lx, absfn, outfn):
71
82
  (lx, absfn, val, len(u''.join(ntext)))
72
83
  tokens.append((type, val))
73
84
  if u''.join(ntext) != text:
74
- print '\n'.join(difflib.unified_diff(u''.join(ntext).splitlines(),
75
- text.splitlines()))
85
+ print('\n'.join(difflib.unified_diff(u''.join(ntext).splitlines(),
86
+ text.splitlines())))
76
87
  raise AssertionError('round trip failed for ' + absfn)
77
88
 
78
89
  # check output against previous run if enabled
@@ -94,6 +105,6 @@ def check_lexer(lx, absfn, outfn):
94
105
  if stored_tokens != tokens:
95
106
  f1 = pprint.pformat(stored_tokens)
96
107
  f2 = pprint.pformat(tokens)
97
- print '\n'.join(difflib.unified_diff(f1.splitlines(),
98
- f2.splitlines()))
108
+ print('\n'.join(difflib.unified_diff(f1.splitlines(),
109
+ f2.splitlines())))
99
110
  assert False, absfn
@@ -3,27 +3,29 @@
3
3
  Pygments HTML formatter tests
4
4
  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5
5
 
6
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
6
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
7
7
  :license: BSD, see LICENSE for details.
8
8
  """
9
9
 
10
+ from __future__ import print_function
11
+
12
+ import io
10
13
  import os
11
14
  import re
12
15
  import unittest
13
- import StringIO
14
16
  import tempfile
15
17
  from os.path import join, dirname, isfile
16
18
 
19
+ from pygments.util import StringIO
17
20
  from pygments.lexers import PythonLexer
18
21
  from pygments.formatters import HtmlFormatter, NullFormatter
19
22
  from pygments.formatters.html import escape_html
20
- from pygments.util import uni_open
21
23
 
22
24
  import support
23
25
 
24
26
  TESTFILE, TESTDIR = support.location(__file__)
25
27
 
26
- fp = uni_open(TESTFILE, encoding='utf-8')
28
+ fp = io.open(TESTFILE, encoding='utf-8')
27
29
  try:
28
30
  tokensource = list(PythonLexer().get_tokens(fp.read()))
29
31
  finally:
@@ -33,11 +35,11 @@ finally:
33
35
  class HtmlFormatterTest(unittest.TestCase):
34
36
  def test_correct_output(self):
35
37
  hfmt = HtmlFormatter(nowrap=True)
36
- houtfile = StringIO.StringIO()
38
+ houtfile = StringIO()
37
39
  hfmt.format(tokensource, houtfile)
38
40
 
39
41
  nfmt = NullFormatter()
40
- noutfile = StringIO.StringIO()
42
+ noutfile = StringIO()
41
43
  nfmt.format(tokensource, noutfile)
42
44
 
43
45
  stripped_html = re.sub('<.*?>', '', houtfile.getvalue())
@@ -74,13 +76,13 @@ class HtmlFormatterTest(unittest.TestCase):
74
76
  dict(linenos=True, full=True),
75
77
  dict(linenos=True, full=True, noclasses=True)]:
76
78
 
77
- outfile = StringIO.StringIO()
79
+ outfile = StringIO()
78
80
  fmt = HtmlFormatter(**optdict)
79
81
  fmt.format(tokensource, outfile)
80
82
 
81
83
  def test_linenos(self):
82
84
  optdict = dict(linenos=True)
83
- outfile = StringIO.StringIO()
85
+ outfile = StringIO()
84
86
  fmt = HtmlFormatter(**optdict)
85
87
  fmt.format(tokensource, outfile)
86
88
  html = outfile.getvalue()
@@ -88,7 +90,7 @@ class HtmlFormatterTest(unittest.TestCase):
88
90
 
89
91
  def test_linenos_with_startnum(self):
90
92
  optdict = dict(linenos=True, linenostart=5)
91
- outfile = StringIO.StringIO()
93
+ outfile = StringIO()
92
94
  fmt = HtmlFormatter(**optdict)
93
95
  fmt.format(tokensource, outfile)
94
96
  html = outfile.getvalue()
@@ -96,7 +98,7 @@ class HtmlFormatterTest(unittest.TestCase):
96
98
 
97
99
  def test_lineanchors(self):
98
100
  optdict = dict(lineanchors="foo")
99
- outfile = StringIO.StringIO()
101
+ outfile = StringIO()
100
102
  fmt = HtmlFormatter(**optdict)
101
103
  fmt.format(tokensource, outfile)
102
104
  html = outfile.getvalue()
@@ -104,7 +106,7 @@ class HtmlFormatterTest(unittest.TestCase):
104
106
 
105
107
  def test_lineanchors_with_startnum(self):
106
108
  optdict = dict(lineanchors="foo", linenostart=5)
107
- outfile = StringIO.StringIO()
109
+ outfile = StringIO()
108
110
  fmt = HtmlFormatter(**optdict)
109
111
  fmt.format(tokensource, outfile)
110
112
  html = outfile.getvalue()
@@ -132,7 +134,7 @@ class HtmlFormatterTest(unittest.TestCase):
132
134
  pass
133
135
  else:
134
136
  if ret:
135
- print output
137
+ print(output)
136
138
  self.assertFalse(ret, 'nsgmls run reported errors')
137
139
 
138
140
  os.unlink(pathname)
@@ -172,7 +174,7 @@ class HtmlFormatterTest(unittest.TestCase):
172
174
  # anymore in the actual source
173
175
  fmt = HtmlFormatter(tagsfile='support/tags', lineanchors='L',
174
176
  tagurlformat='%(fname)s%(fext)s')
175
- outfile = StringIO.StringIO()
177
+ outfile = StringIO()
176
178
  fmt.format(tokensource, outfile)
177
179
  self.assertTrue('<a href="test_html_formatter.py#L-165">test_ctags</a>'
178
180
  in outfile.getvalue())
@@ -3,10 +3,12 @@
3
3
  Pygments LaTeX formatter tests
4
4
  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5
5
 
6
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
6
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
7
7
  :license: BSD, see LICENSE for details.
8
8
  """
9
9
 
10
+ from __future__ import print_function
11
+
10
12
  import os
11
13
  import unittest
12
14
  import tempfile
@@ -48,7 +50,7 @@ class LatexFormatterTest(unittest.TestCase):
48
50
  pass
49
51
  else:
50
52
  if ret:
51
- print output
53
+ print(output)
52
54
  self.assertFalse(ret, 'latex run reported errors')
53
55
 
54
56
  os.unlink(pathname)
@@ -3,7 +3,7 @@
3
3
  Tests for other lexers
4
4
  ~~~~~~~~~~~~~~~~~~~~~~
5
5
 
6
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
6
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
7
7
  :license: BSD, see LICENSE for details.
8
8
  """
9
9
 
@@ -28,7 +28,7 @@ class AnalyseTextTest(unittest.TestCase):
28
28
  for exampleFilePath in glob.glob(exampleFilesPattern):
29
29
  exampleFile = open(exampleFilePath, 'rb')
30
30
  try:
31
- text = exampleFile.read()
31
+ text = exampleFile.read().decode('utf-8')
32
32
  probability = lexer.analyse_text(text)
33
33
  self.assertTrue(probability > 0,
34
34
  '%s must recognize %r' % (
@@ -49,20 +49,20 @@ class RexxLexerTest(unittest.TestCase):
49
49
  self.assertAlmostEqual(1.0,
50
50
  RexxLexer.analyse_text('''/* Rexx */
51
51
  say "hello world"'''))
52
- self.assertLess(0.5,
53
- RexxLexer.analyse_text('/* */\n'
52
+ val = RexxLexer.analyse_text('/* */\n'
54
53
  'hello:pRoceduRe\n'
55
- ' say "hello world"'))
56
- self.assertLess(0.2,
57
- RexxLexer.analyse_text('''/* */
54
+ ' say "hello world"')
55
+ self.assertTrue(val > 0.5, val)
56
+ val = RexxLexer.analyse_text('''/* */
58
57
  if 1 > 0 then do
59
58
  say "ok"
60
59
  end
61
60
  else do
62
61
  say "huh?"
63
- end'''))
64
- self.assertLess(0.2,
65
- RexxLexer.analyse_text('''/* */
62
+ end''')
63
+ self.assertTrue(val > 0.2, val)
64
+ val = RexxLexer.analyse_text('''/* */
66
65
  greeting = "hello world!"
67
66
  parse value greeting "hello" name "!"
68
- say name'''))
67
+ say name''')
68
+ self.assertTrue(val > 0.2, val)
@@ -0,0 +1,81 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Basic CLexer Test
4
+ ~~~~~~~~~~~~~~~~~
5
+
6
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
7
+ :license: BSD, see LICENSE for details.
8
+ """
9
+
10
+ import unittest
11
+ import os
12
+
13
+ from pygments.token import Token
14
+ from pygments.lexers import ObjectiveCLexer
15
+
16
+
17
+ class ObjectiveCLexerTest(unittest.TestCase):
18
+
19
+ def setUp(self):
20
+ self.lexer = ObjectiveCLexer()
21
+
22
+ def testLiteralNumberInt(self):
23
+ fragment = u'@(1);\n'
24
+ expected = [
25
+ (Token.Literal, u'@('),
26
+ (Token.Literal.Number.Integer, u'1'),
27
+ (Token.Literal, u')'),
28
+ (Token.Punctuation, u';'),
29
+ (Token.Text, u'\n'),
30
+ ]
31
+ self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
32
+
33
+ def testLiteralNumberExpression(self):
34
+ fragment = u'@(1+2);\n'
35
+ expected = [
36
+ (Token.Literal, u'@('),
37
+ (Token.Literal.Number.Integer, u'1'),
38
+ (Token.Operator, u'+'),
39
+ (Token.Literal.Number.Integer, u'2'),
40
+ (Token.Literal, u')'),
41
+ (Token.Punctuation, u';'),
42
+ (Token.Text, u'\n'),
43
+ ]
44
+ self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
45
+
46
+ def testLiteralNumberNestedExpression(self):
47
+ fragment = u'@(1+(2+3));\n'
48
+ expected = [
49
+ (Token.Literal, u'@('),
50
+ (Token.Literal.Number.Integer, u'1'),
51
+ (Token.Operator, u'+'),
52
+ (Token.Punctuation, u'('),
53
+ (Token.Literal.Number.Integer, u'2'),
54
+ (Token.Operator, u'+'),
55
+ (Token.Literal.Number.Integer, u'3'),
56
+ (Token.Punctuation, u')'),
57
+ (Token.Literal, u')'),
58
+ (Token.Punctuation, u';'),
59
+ (Token.Text, u'\n'),
60
+ ]
61
+ self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
62
+
63
+ def testLiteralNumberBool(self):
64
+ fragment = u'@NO;\n'
65
+ expected = [
66
+ (Token.Literal.Number, u'@NO'),
67
+ (Token.Punctuation, u';'),
68
+ (Token.Text, u'\n'),
69
+ ]
70
+ self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
71
+
72
+ def testLieralNumberBoolExpression(self):
73
+ fragment = u'@(YES);\n'
74
+ expected = [
75
+ (Token.Literal, u'@('),
76
+ (Token.Name.Builtin, u'YES'),
77
+ (Token.Literal, u')'),
78
+ (Token.Punctuation, u';'),
79
+ (Token.Text, u'\n'),
80
+ ]
81
+ self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
@@ -3,7 +3,7 @@
3
3
  Pygments regex lexer tests
4
4
  ~~~~~~~~~~~~~~~~~~~~~~~~~~
5
5
 
6
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
6
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
7
7
  :license: BSD, see LICENSE for details.
8
8
  """
9
9
 
@@ -0,0 +1,43 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Tests for QBasic
4
+ ~~~~~~~~~~~~~~~~
5
+
6
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
7
+ :license: BSD, see LICENSE for details.
8
+ """
9
+
10
+ import glob
11
+ import os
12
+ import unittest
13
+
14
+ from pygments.token import Token
15
+ from pygments.lexers.qbasic import QBasicLexer
16
+
17
+ class QBasicTest(unittest.TestCase):
18
+ def setUp(self):
19
+ self.lexer = QBasicLexer()
20
+ self.maxDiff = None
21
+
22
+ def testKeywordsWithDollar(self):
23
+ fragment = u'DIM x\nx = RIGHT$("abc", 1)\n'
24
+ expected = [
25
+ (Token.Keyword.Declaration, u'DIM'),
26
+ (Token.Text.Whitespace, u' '),
27
+ (Token.Name.Variable.Global, u'x'),
28
+ (Token.Text, u'\n'),
29
+ (Token.Name.Variable.Global, u'x'),
30
+ (Token.Text.Whitespace, u' '),
31
+ (Token.Operator, u'='),
32
+ (Token.Text.Whitespace, u' '),
33
+ (Token.Keyword.Reserved, u'RIGHT$'),
34
+ (Token.Punctuation, u'('),
35
+ (Token.Literal.String.Double, u'"abc"'),
36
+ (Token.Punctuation, u','),
37
+ (Token.Text.Whitespace, u' '),
38
+ (Token.Literal.Number.Integer.Long, u'1'),
39
+ (Token.Punctuation, u')'),
40
+ (Token.Text, u'\n'),
41
+ ]
42
+ self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
43
+
@@ -3,7 +3,7 @@
3
3
  Pygments regex lexer tests
4
4
  ~~~~~~~~~~~~~~~~~~~~~~~~~~
5
5
 
6
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
6
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
7
7
  :license: BSD, see LICENSE for details.
8
8
  """
9
9
 
@@ -12,6 +12,7 @@ import unittest
12
12
  from pygments.token import Text
13
13
  from pygments.lexer import RegexLexer
14
14
  from pygments.lexer import bygroups
15
+ from pygments.lexer import default
15
16
 
16
17
 
17
18
  class TestLexer(RegexLexer):
@@ -20,6 +21,7 @@ class TestLexer(RegexLexer):
20
21
  'root': [
21
22
  ('a', Text.Root, 'rag'),
22
23
  ('e', Text.Root),
24
+ default(('beer', 'beer'))
23
25
  ],
24
26
  'beer': [
25
27
  ('d', Text.Beer, ('#pop', '#pop')),
@@ -45,3 +47,8 @@ class TupleTransTest(unittest.TestCase):
45
47
  self.assertEqual(toks,
46
48
  [(0, Text.Root, 'a'), (1, Text, u'\n'),
47
49
  (2, Text.Root, 'e')])
50
+
51
+ def test_default(self):
52
+ lx = TestLexer()
53
+ toks = list(lx.get_tokens_unprocessed('d'))
54
+ self.assertEqual(toks, [(0, Text.Beer, 'd')])