pygments.rb 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82a1b6e9549afec22e94b2690983b55a5a3029e986b3d7201c2c6558a2968bc5
-  data.tar.gz: 63ac418bfb998653fc4472c1cfd3c8268a006b002754b1759bc11132889f3e2b
+  metadata.gz: a81b37936d77daa0478878031e065f4c4248ec09f8e5aad0955cb9d3906fdde2
+  data.tar.gz: f4eeaa4c16489986301798f0d760c924b8c5dbe600fe915b3d41607ead341287
 SHA512:
-  metadata.gz: a54ab6bc824be2d4fcdd0c61e52d0838b55ac7cc5ed023f8392ff784fa79d3f1fc32b159c422b46cc91646a8d07853ce2d934fd03e8c06de8903901f45a589c9
-  data.tar.gz: e3910502b53d5c3f6ad729bc558e1baffdbc8af8958fa8867e7489df10bbfd1d1e832f0791a90592418b6bf1c19ba3aa0289b7f043f81aabb071fc036fec29ad
+  metadata.gz: 764c90e3afefa3914a4c9dc21e0bef73f6ab94c87a30aa86e56e4080166a83f943bdd0058fc204ae878f7d2ccf7f23bc849f1363fce510f1062babe8aff66fb5
+  data.tar.gz: f90e47da1207df3ff7913f0acdee8ddd11438015c0664030908f42587bfd97ef67eedae6dcb0f668c2955c1022f3a8fe2127195b339618a4925b4632f0926710

data/.github/workflows/ci.yml

@@ -5,6 +5,18 @@ on:
     tags-ignore: [ '**' ]
   pull_request:
 jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.0'
+          bundler-cache: true
+      - name: Rubocop
+        run: bundle exec rubocop
   build:
     strategy:
       matrix:

data/.rubocop.yml

@@ -0,0 +1,18 @@
+Metrics/AbcSize:
+  Enabled: false
+Metrics/BlockLength:
+  Enabled: false
+Metrics/ClassLength:
+  Enabled: false
+Metrics/CyclomaticComplexity:
+  Enabled: false
+Layout/LineLength:
+  Max: 120
+Metrics/MethodLength:
+  Enabled: false
+Security/MarshalLoad:
+  Enabled: false
+Style/StructInheritance:
+  Enabled: false
+Style/Documentation:
+  Enabled: false

data/CHANGELOG.adoc

@@ -5,6 +5,10 @@
 This document provides a high-level view of the changes to the {project-name} by release.
 For a detailed view of what has changed, refer to the {uri-repo}/commits/master[commit history] on GitHub.
 
+== 2.1.0 (2021-02-14) - @slonopotamus
+
+* Update Pygments to 2.8.0
+
 == 2.0.0 (2021-01-08) - @slonopotamus
 
 * stop sending/receiving `ids` between Ruby and Python

data/Rakefile

@@ -9,8 +9,6 @@ task default: :test
 # Packaging
 # ==========================================================
 
-GEMSPEC = eval(File.read('pygments.rb.gemspec'))
-
 require 'rubygems/package_task'
 
 # ==========================================================
@@ -36,7 +34,7 @@ end
 
 # Write all the lexers to a file for easy lookup
 task :lexers do
-  sh 'ruby cache-lexers.rb'
+  sh 'ruby cache_lexers.rb'
 end
 
 task(:test).enhance([:lexers])

data/bench.rb

@@ -3,7 +3,6 @@
 require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
 require 'benchmark'
 
-include Benchmark
 # number of iterations
 num = ARGV[0] ? ARGV[0].to_i : 10
 
@@ -17,7 +16,13 @@ puts 'Size: ' + code.bytesize.to_s + " bytes\n"
 puts 'Iterations: ' + num.to_s + "\n"
 
 Benchmark.bm(40) do |x|
-  x.report('pygments popen                             ')  { (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); } }
-  x.report('pygments popen (process already started)   ')  { (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); } }
-  x.report('pygments popen (process already started 2) ')  { (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); } }
+  x.report('pygments popen                             ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
+  x.report('pygments popen (process already started)   ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
+  x.report('pygments popen (process already started 2) ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
 end

data/lib/pygments/lexer.rb

@@ -30,7 +30,7 @@ module Pygments
         extnames = []
 
         extname = File.extname(filename)
-        if m = extname.match(/\[(.+)\]/)
+        if (m = extname.match(/\[(.+)\]/))
           m[1].scan(/./).each do |s|
             extnames << extname.sub(m[0], s)
           end

data/lib/pygments/popen.rb

@@ -13,23 +13,13 @@ end
 # Python process.
 module Pygments
   class Popen
-    def popen4(argv)
-      stdin, stdout, stderr, wait_thr = Open3.popen3(*argv, { close_others: true })
-      while (pid = wait_thr.pid).nil? && wait_thr.alive?
-        # For unknown reasons, wait_thr.pid is not immediately available on JRuby
-      end
-      [pid, stdin, stdout, stderr]
-    end
-
     # Get things started by opening a pipe to mentos (the freshmaker), a
     # Python process that talks to the Pygments library. We'll talk back and
     # forth across this pipe.
-    def start(pygments_path = File.expand_path('../../vendor/pygments-main', __dir__))
-      begin
-        @log = Logger.new(ENV['MENTOS_LOG'] || File::NULL)
-        @log.level = Logger::INFO
-        @log.datetime_format = '%Y-%m-%d %H:%M '
-      end
+    def start(pygments_path = File.join(__dir__, '..', '..', 'vendor', 'pygments-main'))
+      @log = Logger.new(ENV['MENTOS_LOG'] || File::NULL)
+      @log.level = Logger::INFO
+      @log.datetime_format = '%Y-%m-%d %H:%M '
 
       ENV['PYGMENTS_PATH'] = pygments_path
 
@@ -38,7 +28,7 @@ module Pygments
 
       # A pipe to the mentos python process. #popen4 gives us
       # the pid and three IO objects to write and read.
-      argv = [*python_binary, File.expand_path('mentos.py', __dir__)]
+      argv = [*python_binary, File.join(__dir__, 'mentos.py')]
       @pid, @in, @out, @err = popen4(argv)
       @in.binmode
       @out.binmode
@@ -53,29 +43,6 @@ module Pygments
       @python_bin = python_bin
     end
 
-    # Detect a suitable Python binary to use.
-    def find_python_binary
-      if Gem.win_platform?
-        return %w[py python3 python].first { |py| !which(py).nil? }
-      end
-
-      # On non-Windows platforms, we simply rely on shebang
-      []
-    end
-
-    # Cross platform which command
-    # from http://stackoverflow.com/a/5471032/284795
-    def which(command)
-      exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
-      ENV['PATH'].split(File::PATH_SEPARATOR).each do |dir|
-        exts.each do |ext|
-          path = File.join(dir, "#{command}#{ext}")
-          return path if File.executable?(path) && !File.directory?(path)
-        end
-      end
-      nil
-    end
-
     # Stop the child process by issuing a kill -9.
     #
     # We then call waitpid() with the pid, which waits for that particular
@@ -88,13 +55,15 @@ module Pygments
     # Technically, kill() can also fail with EPERM or EINVAL (wherein
     # the signal isn't sent); but we have permissions, and
     # we're not doing anything invalid here.
+    # @param reason [String]
     def stop(reason)
       unless @pid.nil?
         @log.info "Killing pid: #{@pid}. Reason: #{reason}"
         begin
           Process.kill('KILL', @pid)
           Process.waitpid(@pid)
-        rescue Errno::ESRCH, Errno::ECHILD
+        rescue Errno::ESRCH, Errno::ECHILD => e
+          @log.warn(e)
         end
       end
       @pid = nil
@@ -108,7 +77,7 @@ module Pygments
     # process and the pid has already been re-used) we'll want to raise
     # that as a more informative Mentos exception.
     #
-    # Returns true if the child is alive.
+    # @return [Boolean] true if the child is alive.
     def alive?
       return true if defined?(@pid) && @pid && Process.kill(0, @pid)
 
@@ -121,7 +90,7 @@ module Pygments
 
     # Public: Get an array of available Pygments formatters
     #
-    # Returns an array of formatters.
+    # @return [Array<String>] an array of formatters
     def formatters
       mentos(:get_all_formatters).each_with_object({}) do |(name, desc, aliases), hash|
         # Remove the long-winded and repetitive 'Formatter' suffix
@@ -134,23 +103,27 @@ module Pygments
       end
     end
 
-    # Public: Get all lexers from a serialized array. This avoids needing to spawn
-    # mentos when it's not really needed (e.g., one-off jobs, loading the Rails env, etc).
+    # Get all lexers from a serialized array.
+    # This avoids needing to spawn mentos when it's not really needed
+    # (e.g., one-off jobs, loading the Rails env, etc).
     #
     # Should be preferred to #lexers!
     #
-    # Returns an array of lexers.
+    # @return [Array<String>] an array of lexers
     def lexers
-      lexer_file = File.expand_path('../../lexers', __dir__)
-      raw = File.open(lexer_file, 'rb').read
-      Marshal.load(raw)
-    rescue Errno::ENOENT
-      raise MentosError, 'Error loading lexer file. Was it created and vendored?'
+      lexer_file = File.join(__dir__, '..', '..', 'lexers')
+      begin
+        File.open(lexer_file, 'rb') do |f|
+          Marshal.load(f)
+        end
+      rescue Errno::ENOENT
+        raise MentosError, %(Error loading #{lexer_file}. Was it created and vendored?)
+      end
     end
 
-    # Public: Get back all available lexers from mentos itself
+    # Get back all available lexers from mentos itself
     #
-    # Returns an array of lexers.
+    # @return [Array<String>] an array of lexers
     def lexers!
       mentos(:get_all_lexers).each_with_object({}) do |lxr, hash|
         name = lxr[0]
@@ -163,17 +136,17 @@ module Pygments
       end
     end
 
-    # Public: Return an array of all available filters
+    # @return [Array<String>] an array of all available filters
     def filters
       mentos(:get_all_filters)
     end
 
-    # Public: Return an array of all available styles
+    # @return [Array<String>] an array of all available styles
     def styles
       mentos(:get_all_styles)
     end
 
-    # Public: Return css for highlighted code
+    # @return [String] css for highlighted code
     def css(klass = '', opts = {})
       if klass.is_a?(Hash)
         opts = klass
@@ -182,7 +155,7 @@ module Pygments
       mentos(:css, ['html', klass], opts)
     end
 
-    # Public: Return the name of a lexer.
+    # @return [String, nil] the name of a lexer.
     def lexer_name_for(*args)
       # Pop off the last arg if it's a hash, which becomes our opts
       opts = if args.last.is_a?(Hash)
@@ -198,8 +171,9 @@ module Pygments
 
     # Public: Highlight code.
     #
-    # Takes a first-position argument of the code to be highlighted, and a
-    # second-position hash of various arguments specifying highlighting properties.
+    # Takes a first-position argument of the code to be highlighted,
+    # and a second-position hash of various arguments specifying
+    # highlighting properties.
     #
     # Returns the highlighted string
     # or nil when the request to the Python process timed out.
@@ -216,29 +190,63 @@ module Pygments
 
       # Get back the string from mentos and force encoding if we can
       str = mentos(:highlight, nil, opts, code)
-      if str.respond_to?(:force_encoding)
-        str.force_encoding(opts[:options][:outencoding])
-      end
+      str.force_encoding(opts[:options][:outencoding]) if str.respond_to?(:force_encoding)
       str
     end
 
     private
 
-    def with_watchdog(timeout_time, error_message)
+    # @param argv [Array<String>]
+    def popen4(argv)
+      stdin, stdout, stderr, wait_thr = Open3.popen3(*argv, { close_others: true })
+      while (pid = wait_thr.pid).nil? && wait_thr.alive?
+        # wait_thr.pid is not immediately available on JRuby. Why???
+      end
+      [pid, stdin, stdout, stderr]
+    end
+
+    # Detect a suitable Python binary to use.
+    def find_python_binary
+      return %w[py python3 python].first { |py| !which(py).nil? } if Gem.win_platform?
+
+      # On non-Windows platforms, we simply rely on shebang
+      []
+    end
+
+    # Cross platform which command
+    # from http://stackoverflow.com/a/5471032/284795
+    def which(command)
+      exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+      ENV['PATH'].split(File::PATH_SEPARATOR).each do |dir|
+        exts.each do |ext|
+          path = File.join(dir, "#{command}#{ext}")
+          return path if File.executable?(path) && !File.directory?(path)
+        end
+      end
+      nil
+    end
+
+    # @param timeout [Integer]
+    # @param error_message [String]
+    # @yield
+    def with_watchdog(timeout, error_message)
       state_mutex = Mutex.new
       state = :alive
       wd_cleanup = ConditionVariable.new
 
-      watchdog = timeout_time > 0 ? Thread.new do
-        state_mutex.synchronize do
-          wd_cleanup.wait(state_mutex, timeout_time) if state != :finished
-          if state != :finished
-            @log.error error_message
-            stop error_message
-            state = :timeout
-          end
-        end
-      end : nil
+      watchdog = if timeout.positive?
+                   Thread.new do
+                     state_mutex.synchronize do
+                       wd_cleanup.wait(state_mutex, timeout) if state != :finished
+                       if state != :finished
+                         @log.error error_message
+                         stop error_message
+                         state = :timeout
+                       end
+                     end
+                   end
+                 end
+
       begin
         yield
       ensure
@@ -261,43 +269,16 @@ module Pygments
       # Open the pipe if necessary
       start unless alive?
 
-      # Timeout requests that take too long.
-      # Invalid MENTOS_TIMEOUT results in just using default.
-      timeout_time = kwargs.delete(:timeout)
-      if timeout_time.nil?
-        timeout_time = begin
-                         Integer(ENV['MENTOS_TIMEOUT'])
-                       rescue TypeError
-                         0
-                       end
-      end
-
       # Add metadata to the header and generate it.
-      bytesize = if code
-                   code.bytesize
-                 else
-                   0
-                 end
-
-      kwargs.freeze
-      kwargs = kwargs.merge('bytes' => bytesize)
+      kwargs = kwargs.merge('bytes' => (code.nil? ? 0 : code.bytesize))
       out_header = JSON.generate(method: method, args: args, kwargs: kwargs)
 
       begin
-        res = with_watchdog(timeout_time, "Timeout on a mentos #{method} call") do
-          # Get the size of the header itself and write that.
-          @in.write([out_header.bytesize].pack('N'))
-          @log.info "Size out: #{out_header.bytesize}"
-
-          # mentos is now waiting for the header, and, potentially, code.
-          @in.write(out_header)
-          @log.info "Out header: #{out_header}"
-          @in.write(code) unless code.nil?
-
-          @in.flush
+        timeout = get_timeout(kwargs.delete(:timeout))
+        res = with_watchdog(timeout, "Timeout on a mentos #{method} call") do
+          write_header(out_header, code)
 
           # mentos will now return data to us. First it sends the header.
-
           header_len_bytes = @out.read(4)
           if header_len_bytes.nil?
             raise Errno::EPIPE, %(Failed to read response from Python process on a mentos #{method} call)
@@ -331,49 +312,70 @@ module Pygments
       end
     end
 
+    # @param header [String]
+    # @param code [String, nil]
+    def write_header(header, code)
+      # Get the size of the header itself and write that.
+      @in.write([header.bytesize].pack('N'))
+      @log.info "Size out: #{header.bytesize}"
+
+      # mentos is now waiting for the header, and, potentially, code.
+      @in.write(header)
+      @log.info "Out header: #{header}"
+      @in.write(code) unless code.nil?
+      @in.flush
+    end
+
+    # @param timeout [Integer, nil]
+    # @return [Integer]
+    def get_timeout(timeout)
+      return timeout unless timeout.nil?
+
+      begin
+        Integer(ENV['MENTOS_TIMEOUT'])
+      rescue TypeError
+        0
+      end
+    end
+
     # Based on the header we receive, determine if we need
     # to read more bytes, and read those bytes if necessary.
     #
-    # Returns a result - either highlighted text or metadata.
+    # @param header [String, nil]
+    # @return [String, nil] either highlighted text or metadata.
     def handle_header_and_return(header)
-      if header
-        @log.info "In header: #{header}"
-        header = header_to_json(header)
-        bytes = header[:bytes]
+      raise MentosError, 'No header received back.' if header.nil?
 
-        # Read more bytes (the actual response body)
-        res = @out.read(bytes.to_i)
+      @log.info "In header: #{header}"
+      header = header_to_json(header)
+      bytes = header[:bytes]
 
-        if header[:method] == 'highlight'
-          # Make sure we have a result back; else consider this an error.
-          raise MentosError, 'No highlight result back from mentos.' if res.nil?
-        end
+      # Read more bytes (the actual response body)
+      res = @out.read(bytes.to_i)
 
-        res
-      else
-        raise MentosError, 'No header received back.'
+      if header[:method] == 'highlight'
+        # Make sure we have a result back; else consider this an error.
+        raise MentosError, 'No highlight result back from mentos.' if res.nil?
       end
+
+      res
     end
 
-    # Return the final result for the API. Return Ruby objects for the methods that
-    # want them, text otherwise.
+    # @return Ruby objects for the methods that want them, text otherwise.
     def return_result(res, method)
-      unless method == :lexer_name_for || method == :highlight || method == :css
-        res = JSON.parse(res, symbolize_names: true)
-      end
+      res = JSON.parse(res, symbolize_names: true) unless %i[lexer_name_for highlight css].include?(method)
       res = res.rstrip if res.class == String
       res
     end
 
     # Convert a text header into JSON for easy access.
+    # @param header [String]
+    # @return [JSON]
     def header_to_json(header)
-      header = JSON.parse(header, symbolize_names: true)
+      json = JSON.parse(header, symbolize_names: true)
+      raise MentosError, json[:error] unless json[:error].nil?
 
-      if header[:error]
-        raise MentosError, header[:error]
-      else
-        header
-      end
+      json
     end
   end
 end