pygments.rb 1.2.1 → 2.2.0

data/bench.rb

@@ -1,22 +1,28 @@
+# frozen_string_literal: true
+
 require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
 require 'benchmark'
 
-include Benchmark
 # number of iterations
 num = ARGV[0] ? ARGV[0].to_i : 10
 
 # we can also repeat the code itself
 repeats = ARGV[1] ? ARGV[1].to_i : 1
 
-code = File.open('test/test_data.py').read.to_s * repeats
+code = File.open('test/test_pygments.rb').read.to_s * repeats
 
 puts "Benchmarking....\n"
-puts "Size: " + code.bytesize.to_s + " bytes\n"
-puts "Iterations: " + num.to_s + "\n"
+puts 'Size: ' + code.bytesize.to_s + " bytes\n"
+puts 'Iterations: ' + num.to_s + "\n"
 
 Benchmark.bm(40) do |x|
-  x.report("pygments popen                             ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
-  x.report("pygments popen (process already started)   ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
-  x.report("pygments popen (process already started 2) ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+  x.report('pygments popen                             ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
+  x.report('pygments popen (process already started)   ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
+  x.report('pygments popen (process already started 2) ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
 end
-

data/lib/pygments.rb

@@ -1,29 +1,31 @@
-# coding: utf-8
 # frozen_string_literal: true
-require File.join(File.dirname(__FILE__), 'pygments/popen')
-require 'forwardable'
 
-module Pygments
+require 'forwardable'
 
-  autoload :Lexer, 'pygments/lexer'
+require_relative 'pygments/lexer'
+require_relative 'pygments/popen'
 
+module Pygments
   class << self
     extend Forwardable
 
+    def lexers
+      LexerCache.instance.raw_lexers
+    end
+
     def engine
       Thread.current.thread_variable_get(:pygments_engine) ||
         Thread.current.thread_variable_set(:pygments_engine, Pygments::Popen.new)
     end
 
     def_delegators :engine,
-      :formatters,
-      :lexers,
-      :lexers!,
-      :filters,
-      :styles,
-      :css,
-      :lexer_name_for,
-      :highlight,
-      :start
+                   :formatters,
+                   :lexers!,
+                   :filters,
+                   :styles,
+                   :css,
+                   :lexer_name_for,
+                   :highlight,
+                   :start
   end
 end

data/lib/pygments/lexer.rb

@@ -1,61 +1,14 @@
-# coding: utf-8
 # frozen_string_literal: true
-module Pygments
-  class Lexer < Struct.new(:name, :aliases, :filenames, :mimetypes)
-    @lexers          = []
-    @index           = {}
-    @name_index      = {}
-    @alias_index     = {}
-    @extname_index   = {}
-    @mimetypes_index = {}
-
-    # Internal: Create a new Lexer object
-    #
-    # hash - A hash of attributes
-    #
-    # Returns a Lexer object
-    def self.create(hash)
-      lexer = new(hash[:name], hash[:aliases], hash[:filenames], hash[:mimetypes])
-
-      @lexers << lexer
-
-      @index[lexer.name.downcase] = @name_index[lexer.name] = lexer
-
-      lexer.aliases.each do |name|
-        @alias_index[name] = lexer
-        @index[name.downcase] ||= lexer
-      end
-
-      lexer.filenames.each do |filename|
-        extnames = []
-
-        extname = File.extname(filename)
-        if m = extname.match(/\[(.+)\]/)
-          m[1].scan(/./).each do |s|
-            extnames << extname.sub(m[0], s)
-          end
-        elsif extname != ""
-          extnames << extname
-        end
-
-        extnames.each do |the_extname|
-          @extname_index[the_extname] = lexer
-          @index[the_extname.downcase.sub(/^\./, "")] ||= lexer
-        end
-      end
-
-      lexer.mimetypes.each do |type|
-        @mimetypes_index[type] = lexer
-      end
 
-      lexer
-    end
+require 'singleton'
 
+module Pygments
+  class Lexer < Struct.new(:name, :aliases, :filenames, :mimetypes)
     # Public: Get all Lexers
     #
-    # Returns an Array of Lexers
+    # @return [Array<Lexer>]
     def self.all
-      @lexers
+      LexerCache.instance.lexers
     end
 
     # Public: Look up Lexer by name or alias.
@@ -65,12 +18,15 @@ module Pygments
     #   Lexer.find('Ruby')
     #   => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @return [Lexer, nil]
     def self.find(name)
-      @index[name.to_s.downcase]
+      LexerCache.instance.index[name.to_s.downcase]
     end
 
     # Public: Alias for find.
+    #
+    # @param name [String]
+    # @return [Lexer, nil]
     def self.[](name)
       find(name)
     end
@@ -84,9 +40,10 @@ module Pygments
     #   Lexer.find_by_name('Ruby')
     #   # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param name [String]
+    # @return [Lexer, nil]
     def self.find_by_name(name)
-      @name_index[name]
+      LexerCache.instance.name_index[name]
     end
 
     # Public: Look up Lexer by one of its aliases.
@@ -98,9 +55,10 @@ module Pygments
     #   Lexer.find_by_alias('rb')
     #   # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param name [String]
+    # @return [Lexer, nil]
     def self.find_by_alias(name)
-      @alias_index[name]
+      LexerCache.instance.alias_index[name]
     end
 
     # Public: Look up Lexer by one of it's file extensions.
@@ -112,9 +70,10 @@ module Pygments
     #  Lexer.find_by_extname('.rb')
     #  # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param extname [String]
+    # @return [Lexer, nil]
     def self.find_by_extname(extname)
-      @extname_index[extname]
+      LexerCache.instance.extname_index[extname]
     end
 
     # Public: Look up Lexer by one of it's mime types.
@@ -126,9 +85,10 @@ module Pygments
     #  Lexer.find_by_mimetype('application/x-ruby')
     #  # => #<Lexer name="Ruby">
     #
-    # Returns the Lexer or nil if none was found.
+    # @param type [String]
+    # @return [Lexer, nil]
     def self.find_by_mimetype(type)
-      @mimetypes_index[type]
+      LexerCache.instance.mimetypes_index[type]
     end
 
     # Public: Highlight syntax of text
@@ -142,9 +102,71 @@ module Pygments
       Pygments.highlight(text, options)
     end
 
-    alias_method :==, :equal?
-    alias_method :eql?, :equal?
+    alias == equal?
+    alias eql? equal?
   end
 
-  lexers.values.each { |h| Lexer.create(h) }
+  class LexerCache
+    include Singleton
+
+    # @return [Array<Lexer>]
+    attr_reader(:lexers)
+    # @return [Map<String, Lexer>]
+    attr_reader(:index)
+    # @return [Map<String, Lexer>]
+    attr_reader(:name_index)
+    # @return [Map<String, Lexer]
+    attr_reader(:alias_index)
+    # @return [Map<String, Lexer>]
+    attr_reader(:extname_index)
+    # @return [Map<String, Lexer>]
+    attr_reader(:mimetypes_index)
+
+    attr_reader(:raw_lexers)
+
+    def initialize
+      @lexers = []
+      @index = {}
+      @name_index = {}
+      @alias_index = {}
+      @extname_index = {}
+      @mimetypes_index = {}
+      @raw_lexers = Pygments.lexers!
+
+      @raw_lexers.values.each do |hash|
+        lexer = Lexer.new(hash[:name], hash[:aliases], hash[:filenames], hash[:mimetypes])
+
+        @lexers << lexer
+
+        @index[lexer.name.downcase] = @name_index[lexer.name] = lexer
+
+        lexer.aliases.each do |name|
+          @alias_index[name] = lexer
+          @index[name.downcase] ||= lexer
+        end
+
+        lexer.filenames.each do |filename|
+          extnames = []
+
+          extname = File.extname(filename)
+          if (m = extname.match(/\[(.+)\]/))
+            m[1].scan(/./).each do |s|
+              extnames << extname.sub(m[0], s)
+            end
+          elsif extname != ''
+            extnames << extname
+          end
+
+          extnames.each do |the_extname|
+            @extname_index[the_extname] = lexer
+            @index[the_extname.downcase.sub(/^\./, '')] ||= lexer
+          end
+        end
+
+        lexer.mimetypes.each do |type|
+          @mimetypes_index[type] = lexer
+        end
+      end
+    end
+  end
 end

data/lib/pygments/mentos.py

@@ -1,6 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import struct
 import sys, re, os, signal
 import traceback
 if 'PYGMENTS_PATH' in os.environ:
@@ -9,15 +10,11 @@ if 'PYGMENTS_PATH' in os.environ:
 dirname = os.path.dirname
 
 base_dir = dirname(dirname(dirname(os.path.abspath(__file__))))
-sys.path.append(base_dir + "/vendor")
 sys.path.append(base_dir + "/vendor/pygments-main")
-sys.path.append(base_dir + "/vendor/simplejson")
 
 import pygments
 from pygments import lexers, formatters, styles, filters
 
-from threading import Lock
-
 try:
     import json
 except ImportError:
@@ -27,22 +24,16 @@ def _convert_keys(dictionary):
     if not isinstance(dictionary, dict):
         return dictionary
     return dict((str(k), _convert_keys(v))
-        for k, v in dictionary.items())
+        for k, v in list(dictionary.items()))
 
 def _write_error(error):
     res = {"error": error}
-    out_header = json.dumps(res).encode('utf-8')
-    bits = _get_fixed_bits_from_header(out_header)
-    sys.stdout.write(bits + "\n")
-    sys.stdout.flush()
-    sys.stdout.write(out_header + "\n")
+    out_header_bytes = json.dumps(res).encode('utf-8')
+    sys.stdout.buffer.write(struct.pack('!i', len(out_header_bytes)))
+    sys.stdout.buffer.write(out_header_bytes)
     sys.stdout.flush()
     return
 
-def _get_fixed_bits_from_header(out_header):
-    size = len(out_header)
-    return "".join(map(lambda y:str((size>>y)&1), range(32-1, -1, -1)))
-
 def _signal_handler(signal, frame):
     """
     Handle the signal given in the first argument, exiting gracefully
@@ -163,12 +154,9 @@ class Mentos(object):
                 res = json.dumps(res)
 
             elif method == 'highlight':
-                try:
-                    text = text.decode('utf-8')
-                except UnicodeDecodeError:
-                    # The text may already be encoded
-                    text = text
                 res = self.highlight_text(text, lexer, formatter_name, args, _convert_keys(opts))
+                if type(res) is bytes:
+                    res = res.decode('utf-8')
 
             elif method == 'css':
                 kwargs = _convert_keys(kwargs)
@@ -193,52 +181,21 @@ class Mentos(object):
 
 
     def _send_data(self, res, method):
-
         # Base header. We'll build on this, adding keys as necessary.
         base_header = {"method": method}
 
-        res_bytes = len(res) + 1
-        base_header["bytes"] = res_bytes
+        res_bytes = res.encode("utf-8")
+        bytes = len(res_bytes)
+        base_header["bytes"] = bytes
 
-        out_header = json.dumps(base_header).encode('utf-8')
-
-        # Following the protocol, send over a fixed size represenation of the
-        # size of the JSON header
-        bits = _get_fixed_bits_from_header(out_header)
+        out_header_bytes = json.dumps(base_header).encode('utf-8')
 
         # Send it to Rubyland
-        sys.stdout.write(bits + "\n")
-        sys.stdout.flush()
-
-        # Send the header.
-        sys.stdout.write(out_header + "\n")
+        sys.stdout.buffer.write(struct.pack('!i', len(out_header_bytes)))
+        sys.stdout.buffer.write(out_header_bytes)
+        sys.stdout.buffer.write(res_bytes)
         sys.stdout.flush()
 
-        # Finally, send the result
-        sys.stdout.write(res + "\n")
-        sys.stdout.flush()
-
-
-    def _get_ids(self, text):
-        start_id = text[:8]
-        end_id = text[-8:]
-        return start_id, end_id
-
-    def _check_and_return_text(self, text, start_id, end_id):
-
-        # Sanity check.
-        id_regex = re.compile('[A-Z]{8}')
-
-        if not id_regex.match(start_id) and not id_regex.match(end_id):
-            _write_error("ID check failed. Not an ID.")
-
-        if not start_id == end_id:
-            _write_error("ID check failed. ID's did not match.")
-
-        # Passed the sanity check. Remove the id's and return
-        text = text[10:-10]
-        return text
-
     def _parse_header(self, header):
         method = header["method"]
         args = header.get("args", [])
@@ -256,29 +213,18 @@ class Mentos(object):
         pygmentized, this header will be followed by the text to be pygmentized.
 
         The header is of form:
-        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128, "fd": "8"}
+        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128}
         """
-        lock = Lock()
 
         while True:
-            # The loop begins by reading off a simple 32-arity string
-            # representing an integer of 32 bits. This is the length of
-            # our JSON header.
-            size = sys.stdin.read(32)
+            header_size_bytes = sys.stdin.buffer.read(4)
+            if not header_size_bytes:
+                break
 
-            lock.acquire()
+            header_size = struct.unpack('!i', header_size_bytes)[0]
 
             try:
-                # Read from stdin the amount of bytes we were told to expect.
-                header_bytes = int(size, 2)
-
-                # Sanity check the size
-                size_regex = re.compile('[0-1]{32}')
-                if not size_regex.match(size):
-                    _write_error("Size received is not valid.")
-
-                line = sys.stdin.read(header_bytes)
-
+                line = sys.stdin.buffer.read(header_size).decode('utf-8')
                 header = json.loads(line)
 
                 method, args, kwargs, lexer = self._parse_header(header)
@@ -291,30 +237,18 @@ class Mentos(object):
                 if kwargs:
                     _bytes = kwargs.get("bytes", 0)
 
-                # Read up to the given number bytes (possibly 0)
-                text = sys.stdin.read(_bytes)
-
-                # Sanity check the return.
-                if _bytes:
-                    start_id, end_id = self._get_ids(text)
-                    text = self._check_and_return_text(text, start_id, end_id)
+                # Read up to the given number of *bytes* (not chars) (possibly 0)
+                text = sys.stdin.buffer.read(_bytes).decode('utf-8')
 
                 # Get the actual data from pygments.
                 res = self.get_data(method, lexer, args, kwargs, text)
 
-                # Put back the sanity check values.
-                if method == "highlight":
-                    res = start_id + "  " + res + "  " + end_id
-
                 self._send_data(res, method)
 
             except:
                 tb = traceback.format_exc()
                 _write_error(tb)
 
-            finally:
-                lock.release()
-
 def main():
 
     # Signal handlers to trap signals.
@@ -324,28 +258,7 @@ def main():
         signal.signal(signal.SIGHUP, _signal_handler)
 
     mentos = Mentos()
-
-    if sys.platform == "win32":
-        # disable CRLF
-        import msvcrt
-        msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
-    else:
-        # close fd's inherited from the ruby parent
-        import resource
-        maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
-        if maxfd == resource.RLIM_INFINITY:
-            maxfd = 65536
-
-        for fd in range(3, maxfd):
-            try:
-                os.close(fd)
-            except:
-                pass
-
     mentos.start()
 
 if __name__ == "__main__":
     main()
-
-
-