pygments.rb 0.6.3 → 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4098ca01dc51f86056c5d9b00813932fcec3e55d
+  data.tar.gz: b1b4b997afea29463fe50d1d71f5149e038f44a9
+SHA512:
+  metadata.gz: 6966a5a27fc61a380855b779220a4b699672e5fe5cb40d554a44b6f81cf48034db918513d1a12a410185685053cd42debbf684675706fcae19a656a811ccb61a
+  data.tar.gz: 05175fe08f110359cbbaf62a6324f4eb1e2231679af5cac40129476d55ac5588dc821e805b658976a363a243cbea48bd69b965ef862858cb878dc47b172f36c7

data/CHANGELOG.md

@@ -1,6 +1,17 @@
 changelog
 ===========
 
+Version 1.0.0 (2016/12/11)
+-----------------------------
+
+* Upgrade bundled pygments to 2.2.0-HEAD [#167](https://github.com/tmm1/pygments.rb/pull/167)
+  * This includes **incompatible changes* because of upgrade of pygments.
+    See http://pygments.org/ for details.
+* Relax yajl-ruby dependency to "~> 1.2" [#164](https://github.com/tmm1/pygments.rb/pull/164)
+* Python binary can be configured by `PYTMENTS_RB_PYTHON` env [#168](https://github.com/tmm1/pygments.rb/pull/168)
+* Improved error messages when python binary is missing [#158](https://github.com/tmm1/pygments.rb/pull/158)
+
+
 Version 0.5.4 (Nov 3, 2013)
 -----------------------------
 

data/README.md

@@ -1,15 +1,15 @@
-# pygments.rb
+# pygments.rb [![CircleCI](https://circleci.com/gh/tmm1/pygments.rb.svg?style=svg)](https://circleci.com/gh/tmm1/pygments.rb) [![Gem Version](https://badge.fury.io/rb/pygments.rb.svg)](https://badge.fury.io/rb/pygments.rb)
 
 A Ruby wrapper for the Python [pygments syntax highlighter](http://pygments.org/).
 
-pygments.rb works by talking over a simple pipe to a long-lived 
+pygments.rb works by talking over a simple pipe to a long-lived
 Python child process. This library replaces [github/albino](https://github.com/github/albino),
 as well as a version of pygments.rb that used an embedded Python
-interpreter. 
+interpreter.
 
-Each Ruby process that runs has its own 'personal Python'; 
-for example, 4 Unicorn workers will have one Python process each. 
-If a Python process dies, a new one will be spawned on the next 
+Each Ruby process that runs has its own 'personal Python';
+for example, 4 Unicorn workers will have one Python process each.
+If a Python process dies, a new one will be spawned on the next
 pygments.rb request.
 
 ## system requirements

data/Rakefile

@@ -9,7 +9,7 @@ task :default => :test
 
 GEMSPEC = eval(File.read('pygments.rb.gemspec'))
 
-require 'rubygems/package_task' 
+require 'rubygems/package_task'
 
 # ==========================================================
 # Testing
@@ -47,6 +47,7 @@ namespace :vendor do
     sh "hg clone https://bitbucket.org/birkenfeld/pygments-main #{f.name}"
     sh "hg --repository #{f.name} identify --id > #{f.name}/REVISION"
     rm_rf Dir["#{f.name}/.hg*"]
+    rm_rf Dir["#{f.name}/tests"]
   end
 
   task :clobber do

data/cache-lexers.rb

@@ -4,5 +4,5 @@ require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
 serialized_lexers = Marshal.dump(Pygments.lexers!)
 
 # Write to a file
-File.open("lexers", 'w') { |file| file.write(serialized_lexers) }
+File.open("lexers", 'wb') { |file| file.write(serialized_lexers) }
 

data/circle.yml

@@ -0,0 +1,7 @@
+machine:
+  ruby:
+    version: 2.3.1
+
+test:
+  override:
+    - bundle exec rake test

data/lib/pygments/lexer.rb

@@ -36,9 +36,9 @@ module Pygments
           extnames << extname
         end
 
-        extnames.each do |extname|
-          @extname_index[extname] = lexer
-          @index[extname.downcase.sub(/^\./, "")] ||= lexer
+        extnames.each do |the_extname|
+          @extname_index[the_extname] = lexer
+          @index[the_extname.downcase.sub(/^\./, "")] ||= lexer
         end
       end
 

data/lib/pygments/popen.rb

@@ -20,13 +20,12 @@ module Pygments
     # Python process that talks to the Pygments library. We'll talk back and
     # forth across this pipe.
     def start(pygments_path = File.expand_path('../../../vendor/pygments-main/', __FILE__))
-      is_windows = RUBY_PLATFORM =~ /mswin|mingw/
       begin
-        @log = Logger.new(ENV['MENTOS_LOG'] ||= is_windows ? 'NUL:' : '/dev/null')
+        @log = Logger.new(ENV['MENTOS_LOG'] ||= File::NULL)
         @log.level = Logger::INFO
         @log.datetime_format = "%Y-%m-%d %H:%M "
       rescue
-        @log = Logger.new(is_windows ? 'NUL:' : '/dev/null')
+        @log = Logger.new(File::NULL)
       end
 
       ENV['PYGMENTS_PATH'] = pygments_path
@@ -36,18 +35,32 @@ module Pygments
 
       # A pipe to the mentos python process. #popen4 gives us
       # the pid and three IO objects to write and read.
-      python_path = python_binary(is_windows)
-      script = "#{python_path} #{File.expand_path('../mentos.py', __FILE__)}"
+      script = "#{python_binary} #{File.expand_path('../mentos.py', __FILE__)}"
       @pid, @in, @out, @err = popen4(script)
-      @log.info "[#{Time.now.iso8601}] Starting pid #{@pid.to_s} with fd #{@out.to_i.to_s}."
+      @log.info "Starting pid #{@pid} with fd #{@out.to_i} and python #{python_binary}."
+    end
+
+    def windows?
+      RUBY_PLATFORM =~ /mswin|mingw/
+    end
+
+    def python_binary
+      @python_binary ||= find_python_binary
+    end
+
+    def python_binary=(python_bin)
+      @python_bin = python_bin
     end
 
     # Detect a suitable Python binary to use.
-    def python_binary(is_windows)
-      if is_windows && which('py')
+    # Or return $PYGMENTS_RB_PYTHON if it's exists.
+    def find_python_binary
+      if ENV['PYGMENTS_RB_PYTHON']
+        return which(ENV['PYGMENTS_RB_PYTHON'])
+      elsif windows? && which('py')
         return 'py -2'
       end
-      return which('python2') || 'python'
+      return which('python2') || which('python')
     end
 
     # Cross platform which command
@@ -83,7 +96,7 @@ module Pygments
         rescue Errno::ESRCH, Errno::ECHILD
         end
       end
-      @log.info "[#{Time.now.iso8601}] Killing pid: #{@pid.to_s}. Reason: #{reason}"
+      @log.info "Killing pid: #{@pid.to_s}. Reason: #{reason}"
       @pid = nil
     end
 
@@ -97,7 +110,7 @@ module Pygments
     #
     # Returns true if the child is alive.
     def alive?
-      return true if @pid && Process.kill(0, @pid)
+      return true if defined?(@pid) && @pid && Process.kill(0, @pid)
       false
     rescue Errno::ENOENT, Errno::ESRCH
       false
@@ -123,11 +136,11 @@ module Pygments
 
 
     # Public: Get all lexers from a serialized array. This avoids needing to spawn
-    # mentos when it's not really needed (e.g,. one-off jobs, loading the Rails env, etc).
+    # mentos when it's not really needed (e.g., one-off jobs, loading the Rails env, etc).
     #
     # Should be preferred to #lexers!
     #
-    # Returns an array of lexers
+    # Returns an array of lexers.
     def lexers
       begin
         lexer_file = File.expand_path('../../../lexers', __FILE__)
@@ -140,7 +153,7 @@ module Pygments
 
     # Public: Get back all available lexers from mentos itself
     #
-    # Returns an array of lexers
+    # Returns an array of lexers.
     def lexers!
       mentos(:get_all_lexers).inject(Hash.new) do |hash, lxr|
         name = lxr[0]
@@ -199,7 +212,10 @@ module Pygments
     # Public: Highlight code.
     #
     # Takes a first-position argument of the code to be highlighted, and a
-    # second-position hash of various arguments specifiying highlighting properties.
+    # second-position hash of various arguments specifying highlighting properties.
+    #
+    # Returns the highlighted string
+    # or nil when the request to the Python process timed out.
     def highlight(code, opts={})
       # If the caller didn't give us any code, we have nothing to do,
       # so return right away.
@@ -255,6 +271,8 @@ module Pygments
           # mentos is now waiting for the header, and, potentially, code.
           write_data(out_header, code)
 
+          check_for_error
+
           # mentos will now return data to us. First it sends the header.
           header = get_header
 
@@ -266,7 +284,7 @@ module Pygments
         end
       rescue Timeout::Error
         # If we timeout, we need to clear out the pipe and start over.
-        @log.error "[#{Time.now.iso8601}] Timeout on a mentos #{method} call"
+        @log.error "Timeout on a mentos #{method} call"
         stop "Timeout on mentos #{method} call."
       end
 
@@ -275,11 +293,30 @@ module Pygments
     raise MentosError, "EPIPE"
     end
 
+    def check_for_error
+      return if @err.closed?
+
+      timeout_time = 0.25 # set a very little timeout so that we do not hang the parser
+
+      Timeout::timeout(timeout_time) do
+        error_msg = @err.read
+
+        unless error_msg.empty?
+          @log.error "Error running python script: #{error_msg}"
+          stop "Error running python script: #{error_msg}"
+          raise MentosError, error_msg
+        end
+      end
+    rescue Timeout::Error
+      # during the specified time no error were found
+      @err.close
+    end
+
 
     # Based on the header we receive, determine if we need
     # to read more bytes, and read those bytes if necessary.
     #
-    # Then, do a sanity check wih the ids.
+    # Then, do a sanity check with the ids.
     #
     # Returns a result — either highlighted text or metadata.
     def handle_header_and_return(header, id)
@@ -293,14 +330,14 @@ module Pygments
         if header["method"] == "highlight"
           # Make sure we have a result back; else consider this an error.
           if res.nil?
-            @log.warn "[#{Time.now.iso8601}] No highlight result back from mentos."
+            @log.warn "No highlight result back from mentos."
             stop "No highlight result back from mentos."
             raise MentosError, "No highlight result back from mentos."
           end
 
           # Remove the newline from Python
           res = res[0..-2]
-          @log.info "[#{Time.now.iso8601}] Highlight in process."
+          @log.info "Highlight in process."
 
           # Get the id's
           start_id = res[0..7]
@@ -308,19 +345,19 @@ module Pygments
 
           # Sanity check.
           if not (start_id == id and end_id == id)
-            @log.error "[#{Time.now.iso8601}] ID's did not match. Aborting."
+            @log.error "ID's did not match. Aborting."
             stop "ID's did not match. Aborting."
             raise MentosError, "ID's did not match. Aborting."
           else
             # We're good. Remove the padding
             res = res[10..-11]
-            @log.info "[#{Time.now.iso8601}] Highlighting complete."
+            @log.info "Highlighting complete."
             res
           end
         end
         res
       else
-        @log.error "[#{Time.now.iso8601}] No header data back."
+        @log.error "No header data back."
         stop "No header data back."
         raise MentosError, "No header received back."
       end
@@ -336,12 +373,12 @@ module Pygments
       code
     end
 
-    # Write data to mentos, the Python Process.
+    # Write data to mentos, the Python process.
     #
     # Returns nothing.
     def write_data(out_header, code=nil)
       @in.write(out_header)
-      @log.info "[#{Time.now.iso8601}] Out header: #{out_header.to_s}"
+      @log.info "Out header: #{out_header}"
       @in.write(code) if code
     end
 
@@ -365,7 +402,7 @@ module Pygments
 
         # Sanity check the size
         if not size_check(size)
-          @log.error "[#{Time.now.iso8601}] Size returned from mentos.py invalid."
+          @log.error "Size returned from mentos.py invalid."
           stop "Size returned from mentos.py invalid."
           raise MentosError, "Size returned from mentos.py invalid."
         end
@@ -373,10 +410,10 @@ module Pygments
         # Read the amount of bytes we should be expecting. We first
         # convert the string of bits into an integer.
         header_bytes = size.to_s.to_i(2) + 1
-        @log.info "[#{Time.now.iso8601}] Size in: #{size.to_s} (#{header_bytes.to_s})"
+        @log.info "Size in: #{size.to_s} (#{header_bytes.to_s})"
         @out.read(header_bytes)
       rescue
-        @log.error "[#{Time.now.iso8601}] Failed to get header."
+        @log.error "Failed to get header."
         stop "Failed to get header."
         raise MentosError, "Failed to get header."
       end
@@ -394,13 +431,13 @@ module Pygments
 
     # Convert a text header into JSON for easy access.
     def header_to_json(header)
-      @log.info "[#{Time.now.iso8601}] In header: #{header.to_s} "
+      @log.info "[In header: #{header} "
       header = Yajl.load(header)
 
       if header["error"]
         # Raise this as a Ruby exception of the MentosError class.
         # Stop so we don't leave the pipe in an inconsistent state.
-        @log.error "[#{Time.now.iso8601}] Failed to convert header to JSON."
+        @log.error "Failed to convert header to JSON."
         stop header["error"]
         raise MentosError, header["error"]
       else
@@ -412,7 +449,7 @@ module Pygments
       size = out_header.bytesize
 
       # Fixed 32 bits to represent the int. We return a string
-      # represenation: e.g, "00000000000000000000000000011110"
+      # representation: e.g, "00000000000000000000000000011110"
       Array.new(32) { |i| size[i] }.reverse!.join
     end
   end

data/lib/pygments/version.rb

@@ -1,3 +1,3 @@
 module Pygments
-  VERSION = '0.6.3'
+  VERSION = '1.0.0'
 end

data/pygments.rb.gemspec

@@ -14,9 +14,10 @@ Gem::Specification.new do |s|
   s.email = ['aman@tmm1.net']
   s.license = 'MIT'
 
-  s.add_dependency 'yajl-ruby',   '~> 1.2.0'
+  s.add_dependency 'yajl-ruby',   '~> 1.2'
   s.add_dependency 'posix-spawn', '~> 0.3.6'
   s.add_development_dependency 'rake-compiler', '~> 0.7.6'
+  s.add_development_dependency 'test-unit', '~> 3.0.0'
 
   # s.extensions = ['ext/extconf.rb']
   s.require_paths = ['lib']

data/test/test_pygments.rb

@@ -13,26 +13,26 @@ class PygmentsHighlightTest < Test::Unit::TestCase
 
   def test_highlight_defaults_to_html
     code = P.highlight(RUBY_CODE)
-    assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
+    assert_match '<span class="ch">#!/usr/bin/ruby</span>', code
     assert_equal '<div class', code[0..9]
   end
 
   def test_full_html_highlight
     code = P.highlight(RUBY_CODE)
-    assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
-    assert_equal "<div class=\"highlight\"><pre><span class=\"c1\">#!/usr/bin/ruby</span>\n<span class=\"nb\">puts</span> <span class=\"s1\">&#39;foo&#39;</span>\n</pre></div>", code
+    assert_match '<span class="ch">#!/usr/bin/ruby</span>', code
+    assert_equal "<div class=\"highlight\"><pre><span></span><span class=\"ch\">#!/usr/bin/ruby</span>\n<span class=\"nb\">puts</span> <span class=\"s1\">&#39;foo&#39;</span>\n</pre></div>", code
   end
 
   def test_full_table_highlight
     code = P.highlight(RUBY_CODE, :options => {:linenos => true})
-    assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
-    assert_equal "<table class=\"highlighttable\"><tr><td class=\"linenos\"><div class=\"linenodiv\"><pre>1\n2</pre></div></td><td class=\"code\"><div class=\"highlight\"><pre><span class=\"c1\">#!/usr/bin/ruby</span>\n<span class=\"nb\">puts</span> <span class=\"s1\">&#39;foo&#39;</span>\n</pre></div>\n</td></tr></table>", code
+    assert_match '<span class="ch">#!/usr/bin/ruby</span>', code
+    assert_equal "<table class=\"highlighttable\"><tr><td class=\"linenos\"><div class=\"linenodiv\"><pre>1\n2</pre></div></td><td class=\"code\"><div class=\"highlight\"><pre><span></span><span class=\"ch\">#!/usr/bin/ruby</span>\n<span class=\"nb\">puts</span> <span class=\"s1\">&#39;foo&#39;</span>\n</pre></div>\n</td></tr></table>", code
   end
 
   def test_highlight_works_with_larger_files
     code = P.highlight(REDIS_CODE)
     assert_match 'used_memory_peak_human', code
-    assert_equal 455203, code.bytesize.to_i
+    assert_equal 458511, code.bytesize.to_i
   end
 
   def test_returns_nil_on_timeout
@@ -59,7 +59,7 @@ class PygmentsHighlightTest < Test::Unit::TestCase
   def test_highlight_works_on_utf8_all_chars_automatically
     code = P.highlight('def foo: # ø', :lexer => 'py')
 
-    assert_equal '<div class="highlight"><pre><span clas', code[0,38]
+    assert_equal "<div class=\"highlight\"><pre><span></sp", code[0,38]
   end
 
   def test_highlight_works_with_multiple_utf8
@@ -89,22 +89,22 @@ class PygmentsHighlightTest < Test::Unit::TestCase
 
   def test_highlight_works_with_single_character_input
     code = P.highlight("a")
-    assert_match 'a</span>', code
+    assert_match "a\n</pre>", code
   end
 
   def test_highlight_works_with_trailing_newline
     code = P.highlight(RUBY_CODE_TRAILING_NEWLINE)
-    assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
+    assert_match '<span class="ch">#!/usr/bin/ruby</span>', code
   end
 
   def test_highlight_works_with_multiple_newlines
     code = P.highlight(RUBY_CODE_TRAILING_NEWLINE + "derp\n\n")
-    assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
+    assert_match '<span class="ch">#!/usr/bin/ruby</span>', code
   end
 
   def test_highlight_works_with_trailing_cr
     code = P.highlight(RUBY_CODE_TRAILING_NEWLINE + "\r")
-    assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
+    assert_match '<span class="ch">#!/usr/bin/ruby</span>', code
   end
 
   def test_highlight_still_works_with_invalid_code
@@ -230,7 +230,7 @@ class PygmentsLexerClassTest < Test::Unit::TestCase
     assert_equal P::Lexer['Ruby'], P::Lexer.find_by_mimetype('text/x-ruby')
     assert_equal P::Lexer['JSON'], P::Lexer.find_by_mimetype('application/json')
     assert_equal P::Lexer['Python'], P::Lexer.find_by_mimetype('text/x-python')
-  end
+ end
 end
 
 
@@ -238,19 +238,19 @@ class PygmentsCssTest < Test::Unit::TestCase
   include Pygments
 
   def test_css
-    assert_match /^\.err \{/, P.css
+    assert_match(/^\.err \{/, P.css)
   end
 
   def test_css_prefix
-    assert_match /^\.highlight \.err \{/, P.css('.highlight')
+    assert_match(/^\.highlight \.err \{/, P.css('.highlight'))
   end
 
   def test_css_options
-    assert_match /^\.codeerr \{/, P.css(:classprefix => 'code')
+    assert_match(/^\.codeerr \{/, P.css(:classprefix => 'code'))
   end
 
   def test_css_prefix_and_options
-    assert_match /^\.mycode \.codeerr \{/, P.css('.mycode', :classprefix => 'code')
+    assert_match(/^\.mycode \.codeerr \{/, P.css('.mycode', :classprefix => 'code'))
   end
 
   def test_css_default