pygments.rb 2.0.0.rc3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb7e6901e07778261de7c0a6283696d5c1593ade93565caf32b2870e11477d39
-  data.tar.gz: 36d123e7aa363b8f50dfcd884e4d1f0721e6e75c83966c18d83f505e7bbf0b85
+  metadata.gz: 82a1b6e9549afec22e94b2690983b55a5a3029e986b3d7201c2c6558a2968bc5
+  data.tar.gz: 63ac418bfb998653fc4472c1cfd3c8268a006b002754b1759bc11132889f3e2b
 SHA512:
-  metadata.gz: 6ad9b6ac8dce86c289939ffbe3ee8c71268dd04ed3755798f08c75905cac90bf602236ae10448a44162809483ca5f88252985db7537d01818c5dcd348d60fe78
-  data.tar.gz: 728ca37e69f29ee9c74d2804e7583fdaf2284f330b9f3096612a88c2cc58833f49ad9a642eadd63e883a36cb7ea171b7f7892a205da709be67937e698a1d49f2
+  metadata.gz: a54ab6bc824be2d4fcdd0c61e52d0838b55ac7cc5ed023f8392ff784fa79d3f1fc32b159c422b46cc91646a8d07853ce2d934fd03e8c06de8903901f45a589c9
+  data.tar.gz: e3910502b53d5c3f6ad729bc558e1baffdbc8af8958fa8867e7489df10bbfd1d1e832f0791a90592418b6bf1c19ba3aa0289b7f043f81aabb071fc036fec29ad

data/CHANGELOG.adoc

@@ -5,6 +5,11 @@
 This document provides a high-level view of the changes to the {project-name} by release.
 For a detailed view of what has changed, refer to the {uri-repo}/commits/master[commit history] on GitHub.
 
+== 2.0.0 (2021-01-08) - @slonopotamus
+
+* stop sending/receiving `ids` between Ruby and Python
+* use `close_others` Ruby mechanism to prevent file descriptor leaking to Python
+
 == 2.0.0.rc3 (2021-01-08) - @slonopotamus
 
 * fix watchdog race condition leading to `ThreadError(<killed thread>)` on JRuby ({uri-repo}/pull/215[#215])

data/README.adoc

@@ -1,7 +1,6 @@
 = {project-name}
 Ted Nyman <ted@ted.io>; Aman Gupta <aman@tmm1.net>; Marat Radchenko <marat@slonopotamus.org>
 :project-name: pygments.rb
-:project-handle: pygments.rb
 :slug: pygments/{project-name}
 :toc: preamble
 :uri-project: https://github.com/{slug}
@@ -14,7 +13,7 @@ image:{uri-project}/workflows/CI/badge.svg?branch=master[Build Status,link={uri-
 
 == Introduction
 
-{project-name} is a Ruby wrapper for the Python {uri-pygments}[Pygments] syntax highlighter.
+{project-name} is a Ruby wrapper for {uri-pygments}[Pygments] syntax highlighter.
 
 {project-name} works by talking over a simple pipe to a long-lived Python child process.
 This library replaces https://github.com/github/albino[github/albino], as well as an older version of {project-name} that used an embedded Python interpreter.

data/lib/pygments/mentos.py

@@ -196,27 +196,6 @@ class Mentos(object):
         sys.stdout.buffer.write(res_bytes)
         sys.stdout.flush()
 
-
-    def _get_ids(self, text):
-        start_id = text[:8]
-        end_id = text[-8:]
-        return start_id, end_id
-
-    def _check_and_return_text(self, text, start_id, end_id):
-
-        # Sanity check.
-        id_regex = re.compile('[A-Z]{8}')
-
-        if not id_regex.match(start_id) and not id_regex.match(end_id):
-            _write_error("ID check failed. Not an ID.")
-
-        if not start_id == end_id:
-            _write_error("ID check failed. ID's did not match.")
-
-        # Passed the sanity check. Remove the id's and return
-        text = text[10:-10]
-        return text
-
     def _parse_header(self, header):
         method = header["method"]
         args = header.get("args", [])
@@ -234,7 +213,7 @@ class Mentos(object):
         pygmentized, this header will be followed by the text to be pygmentized.
 
         The header is of form:
-        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128, "fd": "8"}
+        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128}
         """
 
         while True:
@@ -261,18 +240,9 @@ class Mentos(object):
                 # Read up to the given number of *bytes* (not chars) (possibly 0)
                 text = sys.stdin.buffer.read(_bytes).decode('utf-8')
 
-                # Sanity check the return.
-                if _bytes:
-                    start_id, end_id = self._get_ids(text)
-                    text = self._check_and_return_text(text, start_id, end_id)
-
                 # Get the actual data from pygments.
                 res = self.get_data(method, lexer, args, kwargs, text)
 
-                # Put back the sanity check values.
-                if method == "highlight":
-                    res = start_id + "  " + res + "  " + end_id
-
                 self._send_data(res, method)
 
             except:
@@ -288,20 +258,6 @@ def main():
         signal.signal(signal.SIGHUP, _signal_handler)
 
     mentos = Mentos()
-
-    if sys.platform != "win32":
-        # close fd's inherited from the ruby parent
-        import resource
-        maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
-        if maxfd == resource.RLIM_INFINITY:
-            maxfd = 65536
-
-        for fd in range(3, maxfd):
-            try:
-                os.close(fd)
-            except:
-                pass
-
     mentos.start()
 
 if __name__ == "__main__":

data/lib/pygments/popen.rb

@@ -14,7 +14,7 @@ end
 module Pygments
   class Popen
     def popen4(argv)
-      stdin, stdout, stderr, wait_thr = Open3.popen3(*argv)
+      stdin, stdout, stderr, wait_thr = Open3.popen3(*argv, { close_others: true })
       while (pid = wait_thr.pid).nil? && wait_thr.alive?
         # For unknown reasons, wait_thr.pid is not immediately available on JRuby
       end
@@ -42,7 +42,7 @@ module Pygments
       @pid, @in, @out, @err = popen4(argv)
       @in.binmode
       @out.binmode
-      @log.info "Starting pid #{@pid} with fd #{@out.to_i} and python #{python_binary}."
+      @log.info "Starting pid #{@pid} with python #{python_binary}."
     end
 
     def python_binary
@@ -237,7 +237,6 @@ module Pygments
             stop error_message
             state = :timeout
           end
-
         end
       end : nil
       begin
@@ -258,7 +257,7 @@ module Pygments
 
     # Our 'rpc'-ish request to mentos. Requires a method name, and then optional
     # args, kwargs, code.
-    def mentos(method, args = [], kwargs = {}, original_code = nil)
+    def mentos(method, args = [], kwargs = {}, code = nil)
       # Open the pipe if necessary
       start unless alive?
 
@@ -273,12 +272,6 @@ module Pygments
                        end
       end
 
-      # For sanity checking on both sides of the pipe when highlighting, we prepend and
-      # append an id.  mentos checks that these are 8 character ids and that they match.
-      # It then returns the id's back to Rubyland.
-      id = (0...8).map { rand(65..89).chr }.join
-      code = original_code ? add_ids(original_code, id) : nil
-
       # Add metadata to the header and generate it.
       bytesize = if code
                    code.bytesize
@@ -287,7 +280,7 @@ module Pygments
                  end
 
       kwargs.freeze
-      kwargs = kwargs.merge('fd' => @out.to_i, 'id' => id, 'bytes' => bytesize)
+      kwargs = kwargs.merge('bytes' => bytesize)
       out_header = JSON.generate(method: method, args: args, kwargs: kwargs)
 
       begin
@@ -315,7 +308,7 @@ module Pygments
           header = @out.read(header_len)
 
           # Now handle the header, any read any more data required.
-          handle_header_and_return(header, id)
+          handle_header_and_return(header)
         end
 
         # Finally, return what we got.
@@ -341,10 +334,8 @@ module Pygments
     # Based on the header we receive, determine if we need
     # to read more bytes, and read those bytes if necessary.
     #
-    # Then, do a sanity check with the ids.
-    #
     # Returns a result - either highlighted text or metadata.
-    def handle_header_and_return(header, id)
+    def handle_header_and_return(header)
       if header
         @log.info "In header: #{header}"
         header = header_to_json(header)
@@ -356,37 +347,14 @@ module Pygments
         if header[:method] == 'highlight'
           # Make sure we have a result back; else consider this an error.
           raise MentosError, 'No highlight result back from mentos.' if res.nil?
-
-          @log.info 'Highlight in process.'
-
-          # Get the id's
-          start_id = res[0..7]
-          end_id = res[-8..-1]
-
-          # Sanity check.
-          if !((start_id == id) && (end_id == id))
-            raise MentosError, "ID's did not match. Aborting."
-          else
-            # We're good. Remove the padding
-            res = res[10..-11]
-            @log.info 'Highlighting complete.'
-            res
-          end
         end
+
         res
       else
         raise MentosError, 'No header received back.'
       end
     end
 
-    # With the code, prepend the id (with two spaces to avoid escaping weirdness if
-    # the following text starts with a slash (like terminal code), and append the
-    # id, with two padding also. This means we are sending over the 8 characters +
-    # code + 8 characters.
-    def add_ids(code, id)
-      (id + "  #{code}  #{id}").freeze
-    end
-
     # Return the final result for the API. Return Ruby objects for the methods that
     # want them, text otherwise.
     def return_result(res, method)

data/lib/pygments/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pygments
-  VERSION = '2.0.0.rc3'
+  VERSION = '2.0.0'
 end

data/pygments.rb.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
   s.version = Pygments::VERSION
 
   s.summary = 'pygments wrapper for ruby'
-  s.description = 'pygments.rb exposes the pygments syntax highlighter to Ruby'
+  s.description = 'pygments.rb is a Ruby wrapper for Pygments syntax highlighter'
 
   s.homepage = 'https://github.com/pygments/pygments.rb'
   s.required_ruby_version = '>= 2.3.0'
@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
   s.email = ['marat@slonopotamus.org']
   s.license = 'MIT'
 
-  s.add_development_dependency 'rake-compiler', '~> 1.1.0'
+  s.add_development_dependency 'rake', '~> 13.0.0'
   s.add_development_dependency 'rubocop', '~> 0.81.0'
   s.add_development_dependency 'test-unit', '~> 3.3.0'
 

data/test/test_pygments.rb

@@ -119,31 +119,6 @@ class PygmentsHighlightTest < Test::Unit::TestCase
   end
 end
 
-# Philosophically, I'm not the biggest fan of testing private
-# methods, but given the relative delicacy of validity checking
-# over the pipe I think it's necessary and informative.
-class PygmentsValidityTest < Test::Unit::TestCase
-  def test_add_ids_with_padding
-    res = PE.send(:add_ids, 'herp derp baz boo foo', 'ABCDEFGH')
-    assert_equal 'ABCDEFGH  herp derp baz boo foo  ABCDEFGH', res
-  end
-
-  def test_add_ids_on_empty_string
-    res = PE.send(:add_ids, '', 'ABCDEFGH')
-    assert_equal 'ABCDEFGH    ABCDEFGH', res
-  end
-
-  def test_add_ids_with_unicode_data
-    res = PE.send(:add_ids, '# ø ø ø', 'ABCDEFGH')
-    assert_equal 'ABCDEFGH  # ø ø ø  ABCDEFGH', res
-  end
-
-  def test_add_ids_with_starting_slashes
-    res = PE.send(:add_ids, '\\# ø ø ø..//', 'ABCDEFGH')
-    assert_equal 'ABCDEFGH  \\# ø ø ø..//  ABCDEFGH', res
-  end
-end
-
 class PygmentsLexerTest < Test::Unit::TestCase
   RUBY_CODE = "#!/usr/bin/ruby\nputs 'foo'"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pygments.rb
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc3
+  version: 2.0.0
 platform: ruby
 authors:
 - Aman Gupta
@@ -13,19 +13,19 @@ cert_chain: []
 date: 2021-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rake-compiler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 13.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 13.0.0
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -54,7 +54,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.3.0
-description: pygments.rb exposes the pygments syntax highlighter to Ruby
+description: pygments.rb is a Ruby wrapper for Pygments syntax highlighter
 email:
 - marat@slonopotamus.org
 executables: []
@@ -341,9 +341,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.2.3
 signing_key: