pygments.rb 2.0.0.rc3 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb7e6901e07778261de7c0a6283696d5c1593ade93565caf32b2870e11477d39
-  data.tar.gz: 36d123e7aa363b8f50dfcd884e4d1f0721e6e75c83966c18d83f505e7bbf0b85
+  metadata.gz: 494fdb817ce25f9a10420c22c8dce5c725b157da060f61bad53527af80bb1dff
+  data.tar.gz: 5bb0321b93ea7bac8ee0675803a5bbeee4b74b3784deb0302e50bde0d4dd53e6
 SHA512:
-  metadata.gz: 6ad9b6ac8dce86c289939ffbe3ee8c71268dd04ed3755798f08c75905cac90bf602236ae10448a44162809483ca5f88252985db7537d01818c5dcd348d60fe78
-  data.tar.gz: 728ca37e69f29ee9c74d2804e7583fdaf2284f330b9f3096612a88c2cc58833f49ad9a642eadd63e883a36cb7ea171b7f7892a205da709be67937e698a1d49f2
+  metadata.gz: 363d098055a9887fa9146a2aef00cc5d04929328a55a6c63fca726a34a9b6a4f32d3c4d8f2ba9cf071db23d978fb30868fd6c5d02e2027b1f5bf5529a3394298
+  data.tar.gz: a80e5d794c22e8d700995346c113877b4a8440253d2482578c3393f54f4ffe57f8fcb076731f63a96d1706de19fbeb7c81d42130f9cfbe87c1e06a9783821e8c

data/.github/workflows/ci.yml

@@ -5,11 +5,23 @@ on:
     tags-ignore: [ '**' ]
   pull_request:
 jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.0'
+          bundler-cache: true
+      - name: Rubocop
+        run: bundle exec rubocop
   build:
     strategy:
       matrix:
         ruby: [ 'jruby-9.2', '2.3', '3.0' ]
-        python: [ '3.5', '3.9' ]
+        python: [ '3.5', '3.10' ]
         platform: [ ubuntu-latest, macos-latest, windows-latest ]
     runs-on: ${{ matrix.platform }}
     steps:

data/.gitignore

@@ -3,4 +3,3 @@
 /pkg/
 /tmp
 *.pyc
-/lexers

data/.rubocop.yml

@@ -0,0 +1,16 @@
+Metrics/AbcSize:
+  Enabled: false
+Metrics/BlockLength:
+  Enabled: false
+Metrics/ClassLength:
+  Enabled: false
+Metrics/CyclomaticComplexity:
+  Enabled: false
+Layout/LineLength:
+  Max: 120
+Metrics/MethodLength:
+  Enabled: false
+Style/StructInheritance:
+  Enabled: false
+Style/Documentation:
+  Enabled: false

data/CHANGELOG.adoc

@@ -5,6 +5,25 @@
 This document provides a high-level view of the changes to the {project-name} by release.
 For a detailed view of what has changed, refer to the {uri-repo}/commits/master[commit history] on GitHub.
 
+== 2.3.0 (2022-01-05) - @slonopotamus
+
+* Add `Pygments.pygments_version` method to query underlying Pygments version ({uri-repo}/issues/226[#226])
+* Update Pygments to 2.10.0 ({uri-repo}/issues/234[#234])
+
+== 2.2.0 (2021-03-18) - @slonopotamus
+
+* Add support for custom lexers ({uri-repo}/pull/187[#187])
+* Update Pygments to 2.8.1
+
+== 2.1.0 (2021-02-14) - @slonopotamus
+
+* Update Pygments to 2.8.0
+
+== 2.0.0 (2021-01-08) - @slonopotamus
+
+* stop sending/receiving `ids` between Ruby and Python
+* use `close_others` Ruby mechanism to prevent file descriptor leaking to Python
+
 == 2.0.0.rc3 (2021-01-08) - @slonopotamus
 
 * fix watchdog race condition leading to `ThreadError(<killed thread>)` on JRuby ({uri-repo}/pull/215[#215])

data/README.adoc

@@ -1,9 +1,7 @@
 = {project-name}
 Ted Nyman <ted@ted.io>; Aman Gupta <aman@tmm1.net>; Marat Radchenko <marat@slonopotamus.org>
 :project-name: pygments.rb
-:project-handle: pygments.rb
 :slug: pygments/{project-name}
-:toc: preamble
 :uri-project: https://github.com/{slug}
 :uri-ci: {uri-project}/actions?query=branch%3Amaster
 :uri-gem: https://rubygems.org/gems/{project-name}
@@ -14,7 +12,7 @@ image:{uri-project}/workflows/CI/badge.svg?branch=master[Build Status,link={uri-
 
 == Introduction
 
-{project-name} is a Ruby wrapper for the Python {uri-pygments}[Pygments] syntax highlighter.
+{project-name} is a Ruby wrapper for {uri-pygments}[Pygments] syntax highlighter.
 
 {project-name} works by talking over a simple pipe to a long-lived Python child process.
 This library replaces https://github.com/github/albino[github/albino], as well as an older version of {project-name} that used an embedded Python interpreter.

data/Rakefile

@@ -9,8 +9,6 @@ task default: :test
 # Packaging
 # ==========================================================
 
-GEMSPEC = eval(File.read('pygments.rb.gemspec'))
-
 require 'rubygems/package_task'
 
 # ==========================================================
@@ -30,18 +28,6 @@ task :bench do
   sh 'ruby bench.rb'
 end
 
-# ==========================================================
-# Cache lexers
-# ==========================================================
-
-# Write all the lexers to a file for easy lookup
-task :lexers do
-  sh 'ruby cache-lexers.rb'
-end
-
-task(:test).enhance([:lexers])
-task(:build).enhance([:lexers])
-
 # ==========================================================
 # Vendor
 # ==========================================================

data/bench.rb

@@ -3,7 +3,6 @@
 require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
 require 'benchmark'
 
-include Benchmark
 # number of iterations
 num = ARGV[0] ? ARGV[0].to_i : 10
 
@@ -13,11 +12,17 @@ repeats = ARGV[1] ? ARGV[1].to_i : 1
 code = File.open('test/test_pygments.rb').read.to_s * repeats
 
 puts "Benchmarking....\n"
-puts 'Size: ' + code.bytesize.to_s + " bytes\n"
-puts 'Iterations: ' + num.to_s + "\n"
+puts "Size: #{code.bytesize} bytes\n"
+puts "Iterations: #{num}\n"
 
 Benchmark.bm(40) do |x|
-  x.report('pygments popen                             ')  { (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); } }
-  x.report('pygments popen (process already started)   ')  { (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); } }
-  x.report('pygments popen (process already started 2) ')  { (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); } }
+  x.report('pygments popen                             ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
+  x.report('pygments popen (process already started)   ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
+  x.report('pygments popen (process already started 2) ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
 end

data/lib/pygments/lexer.rb

@@ -1,61 +1,14 @@
 # frozen_string_literal: true
 
+require 'singleton'
+
 module Pygments
   class Lexer < Struct.new(:name, :aliases, :filenames, :mimetypes)
-    @lexers          = []
-    @index           = {}
-    @name_index      = {}
-    @alias_index     = {}
-    @extname_index   = {}
-    @mimetypes_index = {}
-
-    # Internal: Create a new Lexer object
-    #
-    # hash - A hash of attributes
-    #
-    # Returns a Lexer object
-    def self.create(hash)
-      lexer = new(hash[:name], hash[:aliases], hash[:filenames], hash[:mimetypes])
-
-      @lexers << lexer
-
-      @index[lexer.name.downcase] = @name_index[lexer.name] = lexer
-
-      lexer.aliases.each do |name|
-        @alias_index[name] = lexer
-        @index[name.downcase] ||= lexer
-      end
-
-      lexer.filenames.each do |filename|
-        extnames = []
-
-        extname = File.extname(filename)
-        if m = extname.match(/\[(.+)\]/)
-          m[1].scan(/./).each do |s|
-            extnames << extname.sub(m[0], s)
-          end
-        elsif extname != ''
-          extnames << extname
-        end
-
-        extnames.each do |the_extname|
-          @extname_index[the_extname] = lexer
-          @index[the_extname.downcase.sub(/^\./, '')] ||= lexer
-        end
-      end
-
-      lexer.mimetypes.each do |type|
-        @mimetypes_index[type] = lexer
-      end
-
-      lexer
-    end
-
     # Public: Get all Lexers
     #
-    # Returns an Array of Lexers
+    # @return [Array<Lexer>]
     def self.all
-      @lexers
+      LexerCache.instance.lexers
     end
 
     # Public: Look up Lexer by name or alias.
@@ -65,12 +18,15 @@ module Pygments
     #   Lexer.find('Ruby')
     #   => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @return [Lexer, nil]
     def self.find(name)
-      @index[name.to_s.downcase]
+      LexerCache.instance.index[name.to_s.downcase]
     end
 
     # Public: Alias for find.
+    #
+    # @param name [String]
+    # @return [Lexer, nil]
     def self.[](name)
       find(name)
     end
@@ -84,9 +40,10 @@ module Pygments
     #   Lexer.find_by_name('Ruby')
     #   # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param name [String]
+    # @return [Lexer, nil]
     def self.find_by_name(name)
-      @name_index[name]
+      LexerCache.instance.name_index[name]
     end
 
     # Public: Look up Lexer by one of its aliases.
@@ -98,9 +55,10 @@ module Pygments
     #   Lexer.find_by_alias('rb')
     #   # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param name [String]
+    # @return [Lexer, nil]
     def self.find_by_alias(name)
-      @alias_index[name]
+      LexerCache.instance.alias_index[name]
     end
 
     # Public: Look up Lexer by one of it's file extensions.
@@ -112,9 +70,10 @@ module Pygments
     #  Lexer.find_by_extname('.rb')
     #  # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param extname [String]
+    # @return [Lexer, nil]
     def self.find_by_extname(extname)
-      @extname_index[extname]
+      LexerCache.instance.extname_index[extname]
     end
 
     # Public: Look up Lexer by one of it's mime types.
@@ -126,9 +85,10 @@ module Pygments
     #  Lexer.find_by_mimetype('application/x-ruby')
     #  # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param type [String]
+    # @return [Lexer, nil]
     def self.find_by_mimetype(type)
-      @mimetypes_index[type]
+      LexerCache.instance.mimetypes_index[type]
     end
 
     # Public: Highlight syntax of text
@@ -146,5 +106,67 @@ module Pygments
     alias eql? equal?
   end
 
-  lexers.values.each { |h| Lexer.create(h) }
+  class LexerCache
+    include Singleton
+
+    # @return [Array<Lexer>]
+    attr_reader(:lexers)
+    # @return [Map<String, Lexer>]
+    attr_reader(:index)
+    # @return [Map<String, Lexer>]
+    attr_reader(:name_index)
+    # @return [Map<String, Lexer]
+    attr_reader(:alias_index)
+    # @return [Map<String, Lexer>]
+    attr_reader(:extname_index)
+    # @return [Map<String, Lexer>]
+    attr_reader(:mimetypes_index)
+
+    attr_reader(:raw_lexers)
+
+    def initialize
+      @lexers = []
+      @index = {}
+      @name_index = {}
+      @alias_index = {}
+      @extname_index = {}
+      @mimetypes_index = {}
+      @raw_lexers = Pygments.lexers!
+
+      @raw_lexers.each_value do |hash|
+        lexer = Lexer.new(hash[:name], hash[:aliases], hash[:filenames], hash[:mimetypes])
+
+        @lexers << lexer
+
+        @index[lexer.name.downcase] = @name_index[lexer.name] = lexer
+
+        lexer.aliases.each do |name|
+          @alias_index[name] = lexer
+          @index[name.downcase] ||= lexer
+        end
+
+        lexer.filenames.each do |filename|
+          extnames = []
+
+          extname = File.extname(filename)
+          if (m = extname.match(/\[(.+)\]/))
+            m[1].scan(/./).each do |s|
+              extnames << extname.sub(m[0], s)
+            end
+          elsif extname != ''
+            extnames << extname
+          end
+
+          extnames.each do |the_extname|
+            @extname_index[the_extname] = lexer
+            @index[the_extname.downcase.sub(/^\./, '')] ||= lexer
+          end
+        end
+
+        lexer.mimetypes.each do |type|
+          @mimetypes_index[type] = lexer
+        end
+      end
+    end
+  end
 end

data/lib/pygments/mentos.py

@@ -163,17 +163,19 @@ class Mentos(object):
                 fmt = pygments.formatters.get_formatter_by_name(args[0], **kwargs)
                 res = fmt.get_style_defs(args[1])
 
-            elif method == 'lexer_name_for':
+            elif method == 'lexer_names_for':
                 lexer = self.return_lexer(None, args, kwargs, text)
 
                 if lexer:
-                    # We don't want the Lexer itself, just the name.
-                    # Take the first alias.
-                    res = lexer.aliases[0]
+                    # We don't want the Lexer itself, just aliases.
+                    res = json.dumps(list(lexer.aliases))
 
                 else:
                     _write_error("No lexer")
 
+            elif method == 'version':
+                res = json.dumps([pygments.__version__])
+
             else:
                 _write_error("Invalid method " + method)
 
@@ -196,27 +198,6 @@ class Mentos(object):
         sys.stdout.buffer.write(res_bytes)
         sys.stdout.flush()
 
-
-    def _get_ids(self, text):
-        start_id = text[:8]
-        end_id = text[-8:]
-        return start_id, end_id
-
-    def _check_and_return_text(self, text, start_id, end_id):
-
-        # Sanity check.
-        id_regex = re.compile('[A-Z]{8}')
-
-        if not id_regex.match(start_id) and not id_regex.match(end_id):
-            _write_error("ID check failed. Not an ID.")
-
-        if not start_id == end_id:
-            _write_error("ID check failed. ID's did not match.")
-
-        # Passed the sanity check. Remove the id's and return
-        text = text[10:-10]
-        return text
-
     def _parse_header(self, header):
         method = header["method"]
         args = header.get("args", [])
@@ -234,7 +215,7 @@ class Mentos(object):
         pygmentized, this header will be followed by the text to be pygmentized.
 
         The header is of form:
-        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128, "fd": "8"}
+        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128}
         """
 
         while True:
@@ -261,18 +242,9 @@ class Mentos(object):
                 # Read up to the given number of *bytes* (not chars) (possibly 0)
                 text = sys.stdin.buffer.read(_bytes).decode('utf-8')
 
-                # Sanity check the return.
-                if _bytes:
-                    start_id, end_id = self._get_ids(text)
-                    text = self._check_and_return_text(text, start_id, end_id)
-
                 # Get the actual data from pygments.
                 res = self.get_data(method, lexer, args, kwargs, text)
 
-                # Put back the sanity check values.
-                if method == "highlight":
-                    res = start_id + "  " + res + "  " + end_id
-
                 self._send_data(res, method)
 
             except:
@@ -288,20 +260,6 @@ def main():
         signal.signal(signal.SIGHUP, _signal_handler)
 
     mentos = Mentos()
-
-    if sys.platform != "win32":
-        # close fd's inherited from the ruby parent
-        import resource
-        maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
-        if maxfd == resource.RLIM_INFINITY:
-            maxfd = 65536
-
-        for fd in range(3, maxfd):
-            try:
-                os.close(fd)
-            except:
-                pass
-
     mentos.start()
 
 if __name__ == "__main__":