pygments.rb 1.2.0 → 2.1.0

data/bench.rb

@@ -1,22 +1,28 @@
+# frozen_string_literal: true
+
 require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
 require 'benchmark'
 
-include Benchmark
 # number of iterations
 num = ARGV[0] ? ARGV[0].to_i : 10
 
 # we can also repeat the code itself
 repeats = ARGV[1] ? ARGV[1].to_i : 1
 
-code = File.open('test/test_data.py').read.to_s * repeats
+code = File.open('test/test_pygments.rb').read.to_s * repeats
 
 puts "Benchmarking....\n"
-puts "Size: " + code.bytesize.to_s + " bytes\n"
-puts "Iterations: " + num.to_s + "\n"
+puts 'Size: ' + code.bytesize.to_s + " bytes\n"
+puts 'Iterations: ' + num.to_s + "\n"
 
 Benchmark.bm(40) do |x|
-  x.report("pygments popen                             ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
-  x.report("pygments popen (process already started)   ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
-  x.report("pygments popen (process already started 2) ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+  x.report('pygments popen                             ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
+  x.report('pygments popen (process already started)   ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
+  x.report('pygments popen (process already started 2) ') do
+    (1..num).each { |_i|; Pygments.highlight(code, lexer: 'python'); }
+  end
 end
-

data/{cache-lexers.rb → cache_lexers.rb}

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
+
 require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
 
 # Simple marshalling
 serialized_lexers = Marshal.dump(Pygments.lexers!)
 
 # Write to a file
-File.open("lexers", 'wb') { |file| file.write(serialized_lexers) }
-
+File.open('lexers', 'wb') { |file| file.write(serialized_lexers) }

data/lib/pygments.rb

@@ -1,10 +1,9 @@
-# coding: utf-8
 # frozen_string_literal: true
+
 require File.join(File.dirname(__FILE__), 'pygments/popen')
 require 'forwardable'
 
 module Pygments
-
   autoload :Lexer, 'pygments/lexer'
 
   class << self
@@ -16,14 +15,14 @@ module Pygments
     end
 
     def_delegators :engine,
-      :formatters,
-      :lexers,
-      :lexers!,
-      :filters,
-      :styles,
-      :css,
-      :lexer_name_for,
-      :highlight,
-      :start
+                   :formatters,
+                   :lexers,
+                   :lexers!,
+                   :filters,
+                   :styles,
+                   :css,
+                   :lexer_name_for,
+                   :highlight,
+                   :start
   end
 end

data/lib/pygments/lexer.rb

@@ -1,5 +1,5 @@
-# coding: utf-8
 # frozen_string_literal: true
+
 module Pygments
   class Lexer < Struct.new(:name, :aliases, :filenames, :mimetypes)
     @lexers          = []
@@ -30,17 +30,17 @@ module Pygments
         extnames = []
 
         extname = File.extname(filename)
-        if m = extname.match(/\[(.+)\]/)
+        if (m = extname.match(/\[(.+)\]/))
           m[1].scan(/./).each do |s|
             extnames << extname.sub(m[0], s)
           end
-        elsif extname != ""
+        elsif extname != ''
           extnames << extname
         end
 
         extnames.each do |the_extname|
           @extname_index[the_extname] = lexer
-          @index[the_extname.downcase.sub(/^\./, "")] ||= lexer
+          @index[the_extname.downcase.sub(/^\./, '')] ||= lexer
         end
       end
 
@@ -142,8 +142,8 @@ module Pygments
       Pygments.highlight(text, options)
     end
 
-    alias_method :==, :equal?
-    alias_method :eql?, :equal?
+    alias == equal?
+    alias eql? equal?
   end
 
   lexers.values.each { |h| Lexer.create(h) }

data/lib/pygments/mentos.py

@@ -1,6 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import struct
 import sys, re, os, signal
 import traceback
 if 'PYGMENTS_PATH' in os.environ:
@@ -9,15 +10,11 @@ if 'PYGMENTS_PATH' in os.environ:
 dirname = os.path.dirname
 
 base_dir = dirname(dirname(dirname(os.path.abspath(__file__))))
-sys.path.append(base_dir + "/vendor")
 sys.path.append(base_dir + "/vendor/pygments-main")
-sys.path.append(base_dir + "/vendor/simplejson")
 
 import pygments
 from pygments import lexers, formatters, styles, filters
 
-from threading import Lock
-
 try:
     import json
 except ImportError:
@@ -27,22 +24,16 @@ def _convert_keys(dictionary):
     if not isinstance(dictionary, dict):
         return dictionary
     return dict((str(k), _convert_keys(v))
-        for k, v in dictionary.items())
+        for k, v in list(dictionary.items()))
 
 def _write_error(error):
     res = {"error": error}
-    out_header = json.dumps(res).encode('utf-8')
-    bits = _get_fixed_bits_from_header(out_header)
-    sys.stdout.write(bits + "\n")
-    sys.stdout.flush()
-    sys.stdout.write(out_header + "\n")
+    out_header_bytes = json.dumps(res).encode('utf-8')
+    sys.stdout.buffer.write(struct.pack('!i', len(out_header_bytes)))
+    sys.stdout.buffer.write(out_header_bytes)
     sys.stdout.flush()
     return
 
-def _get_fixed_bits_from_header(out_header):
-    size = len(out_header)
-    return "".join(map(lambda y:str((size>>y)&1), range(32-1, -1, -1)))
-
 def _signal_handler(signal, frame):
     """
     Handle the signal given in the first argument, exiting gracefully
@@ -163,12 +154,9 @@ class Mentos(object):
                 res = json.dumps(res)
 
             elif method == 'highlight':
-                try:
-                    text = text.decode('utf-8')
-                except UnicodeDecodeError:
-                    # The text may already be encoded
-                    text = text
                 res = self.highlight_text(text, lexer, formatter_name, args, _convert_keys(opts))
+                if type(res) is bytes:
+                    res = res.decode('utf-8')
 
             elif method == 'css':
                 kwargs = _convert_keys(kwargs)
@@ -193,52 +181,21 @@ class Mentos(object):
 
 
     def _send_data(self, res, method):
-
         # Base header. We'll build on this, adding keys as necessary.
         base_header = {"method": method}
 
-        res_bytes = len(res) + 1
-        base_header["bytes"] = res_bytes
+        res_bytes = res.encode("utf-8")
+        bytes = len(res_bytes)
+        base_header["bytes"] = bytes
 
-        out_header = json.dumps(base_header).encode('utf-8')
-
-        # Following the protocol, send over a fixed size represenation of the
-        # size of the JSON header
-        bits = _get_fixed_bits_from_header(out_header)
+        out_header_bytes = json.dumps(base_header).encode('utf-8')
 
         # Send it to Rubyland
-        sys.stdout.write(bits + "\n")
-        sys.stdout.flush()
-
-        # Send the header.
-        sys.stdout.write(out_header + "\n")
+        sys.stdout.buffer.write(struct.pack('!i', len(out_header_bytes)))
+        sys.stdout.buffer.write(out_header_bytes)
+        sys.stdout.buffer.write(res_bytes)
         sys.stdout.flush()
 
-        # Finally, send the result
-        sys.stdout.write(res + "\n")
-        sys.stdout.flush()
-
-
-    def _get_ids(self, text):
-        start_id = text[:8]
-        end_id = text[-8:]
-        return start_id, end_id
-
-    def _check_and_return_text(self, text, start_id, end_id):
-
-        # Sanity check.
-        id_regex = re.compile('[A-Z]{8}')
-
-        if not id_regex.match(start_id) and not id_regex.match(end_id):
-            _write_error("ID check failed. Not an ID.")
-
-        if not start_id == end_id:
-            _write_error("ID check failed. ID's did not match.")
-
-        # Passed the sanity check. Remove the id's and return
-        text = text[10:-10]
-        return text
-
     def _parse_header(self, header):
         method = header["method"]
         args = header.get("args", [])
@@ -256,29 +213,18 @@ class Mentos(object):
         pygmentized, this header will be followed by the text to be pygmentized.
 
         The header is of form:
-        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128, "fd": "8"}
+        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128}
         """
-        lock = Lock()
 
         while True:
-            # The loop begins by reading off a simple 32-arity string
-            # representing an integer of 32 bits. This is the length of
-            # our JSON header.
-            size = sys.stdin.read(32)
+            header_size_bytes = sys.stdin.buffer.read(4)
+            if not header_size_bytes:
+                break
 
-            lock.acquire()
+            header_size = struct.unpack('!i', header_size_bytes)[0]
 
             try:
-                # Read from stdin the amount of bytes we were told to expect.
-                header_bytes = int(size, 2)
-
-                # Sanity check the size
-                size_regex = re.compile('[0-1]{32}')
-                if not size_regex.match(size):
-                    _write_error("Size received is not valid.")
-
-                line = sys.stdin.read(header_bytes)
-
+                line = sys.stdin.buffer.read(header_size).decode('utf-8')
                 header = json.loads(line)
 
                 method, args, kwargs, lexer = self._parse_header(header)
@@ -291,30 +237,18 @@ class Mentos(object):
                 if kwargs:
                     _bytes = kwargs.get("bytes", 0)
 
-                # Read up to the given number bytes (possibly 0)
-                text = sys.stdin.read(_bytes)
-
-                # Sanity check the return.
-                if _bytes:
-                    start_id, end_id = self._get_ids(text)
-                    text = self._check_and_return_text(text, start_id, end_id)
+                # Read up to the given number of *bytes* (not chars) (possibly 0)
+                text = sys.stdin.buffer.read(_bytes).decode('utf-8')
 
                 # Get the actual data from pygments.
                 res = self.get_data(method, lexer, args, kwargs, text)
 
-                # Put back the sanity check values.
-                if method == "highlight":
-                    res = start_id + "  " + res + "  " + end_id
-
                 self._send_data(res, method)
 
             except:
                 tb = traceback.format_exc()
                 _write_error(tb)
 
-            finally:
-                lock.release()
-
 def main():
 
     # Signal handlers to trap signals.
@@ -324,28 +258,7 @@ def main():
         signal.signal(signal.SIGHUP, _signal_handler)
 
     mentos = Mentos()
-
-    if sys.platform == "win32":
-        # disable CRLF
-        import msvcrt
-        msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
-    else:
-        # close fd's inherited from the ruby parent
-        import resource
-        maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
-        if maxfd == resource.RLIM_INFINITY:
-            maxfd = 65536
-
-        for fd in range(3, maxfd):
-            try:
-                os.close(fd)
-            except:
-                pass
-
     mentos.start()
 
 if __name__ == "__main__":
     main()
-
-
-

data/lib/pygments/popen.rb

@@ -1,9 +1,7 @@
-# coding: utf-8
 # frozen_string_literal: true
 
+require 'json'
 require 'open3'
-require 'multi_json'
-require 'timeout'
 require 'logger'
 require 'time'
 
@@ -15,38 +13,26 @@ end
 # Python process.
 module Pygments
   class Popen
-
-    def popen4(cmd)
-      stdin, stdout, stderr, wait_thr = Open3.popen3(cmd)
-      [wait_thr[:pid], stdin, stdout, stderr]
-    end
-
     # Get things started by opening a pipe to mentos (the freshmaker), a
     # Python process that talks to the Pygments library. We'll talk back and
     # forth across this pipe.
-    def start(pygments_path = File.expand_path('../../../vendor/pygments-main/', __FILE__))
-      begin
-        @log = Logger.new(ENV['MENTOS_LOG'] ||= File::NULL)
-        @log.level = Logger::INFO
-        @log.datetime_format = "%Y-%m-%d %H:%M "
-      rescue
-        @log = Logger.new(File::NULL)
-      end
+    def start(pygments_path = File.join(__dir__, '..', '..', 'vendor', 'pygments-main'))
+      @log = Logger.new(ENV['MENTOS_LOG'] || File::NULL)
+      @log.level = Logger::INFO
+      @log.datetime_format = '%Y-%m-%d %H:%M '
 
       ENV['PYGMENTS_PATH'] = pygments_path
 
       # Make sure we kill off the child when we're done
-      at_exit { stop "Exiting" }
+      at_exit { stop 'Exiting' }
 
       # A pipe to the mentos python process. #popen4 gives us
       # the pid and three IO objects to write and read.
-      script = "#{python_binary} #{File.expand_path('../mentos.py', __FILE__)}"
-      @pid, @in, @out, @err = popen4(script)
-      @log.info "Starting pid #{@pid} with fd #{@out.to_i} and python #{python_binary}."
-    end
-
-    def windows?
-      RUBY_PLATFORM =~ /mswin|mingw/
+      argv = [*python_binary, File.join(__dir__, 'mentos.py')]
+      @pid, @in, @out, @err = popen4(argv)
+      @in.binmode
+      @out.binmode
+      @log.info "Starting pid #{@pid} with python #{python_binary}."
     end
 
     def python_binary
@@ -57,30 +43,6 @@ module Pygments
       @python_bin = python_bin
     end
 
-    # Detect a suitable Python binary to use.
-    # Or return $PYGMENTS_RB_PYTHON if it's exists.
-    def find_python_binary
-      if ENV['PYGMENTS_RB_PYTHON']
-        return which(ENV['PYGMENTS_RB_PYTHON'])
-      elsif windows? && which('py')
-        return 'py -2'
-      end
-      return which('python2') || which('python')
-    end
-
-    # Cross platform which command
-    # from http://stackoverflow.com/a/5471032/284795
-    def which(command)
-      exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
-      ENV['PATH'].split(File::PATH_SEPARATOR).each do |dir|
-        exts.each { |ext|
-          path = File.join(dir, "#{command}#{ext}")
-          return path if File.executable?(path) && !File.directory?(path)
-        }
-      end
-      return nil
-    end
-
     # Stop the child process by issuing a kill -9.
     #
     # We then call waitpid() with the pid, which waits for that particular
@@ -93,15 +55,17 @@ module Pygments
     # Technically, kill() can also fail with EPERM or EINVAL (wherein
     # the signal isn't sent); but we have permissions, and
     # we're not doing anything invalid here.
+    # @param reason [String]
     def stop(reason)
-      if @pid
+      unless @pid.nil?
+        @log.info "Killing pid: #{@pid}. Reason: #{reason}"
         begin
           Process.kill('KILL', @pid)
           Process.waitpid(@pid)
-        rescue Errno::ESRCH, Errno::ECHILD
+        rescue Errno::ESRCH, Errno::ECHILD => e
+          @log.warn(e)
         end
       end
-      @log.info "Killing pid: #{@pid.to_s}. Reason: #{reason}"
       @pid = nil
     end
 
@@ -113,82 +77,77 @@ module Pygments
     # process and the pid has already been re-used) we'll want to raise
     # that as a more informative Mentos exception.
     #
-    # Returns true if the child is alive.
+    # @return [Boolean] true if the child is alive.
     def alive?
       return true if defined?(@pid) && @pid && Process.kill(0, @pid)
+
       false
     rescue Errno::ENOENT, Errno::ESRCH
       false
     rescue Errno::EPERM
-      raise MentosError, "EPERM checking if child process is alive."
+      raise MentosError, 'EPERM checking if child process is alive.'
     end
 
     # Public: Get an array of available Pygments formatters
     #
-    # Returns an array of formatters.
+    # @return [Array<String>] an array of formatters
     def formatters
-      mentos(:get_all_formatters).inject(Hash.new) do | hash, (name, desc, aliases) |
+      mentos(:get_all_formatters).each_with_object({}) do |(name, desc, aliases), hash|
         # Remove the long-winded and repetitive 'Formatter' suffix
         name.sub!(/Formatter$/, '')
         hash[name] = {
-          :name => name,
-          :description => desc,
-          :aliases => aliases
+          name: name,
+          description: desc,
+          aliases: aliases
         }
-        hash
       end
     end
 
-
-    # Public: Get all lexers from a serialized array. This avoids needing to spawn
-    # mentos when it's not really needed (e.g., one-off jobs, loading the Rails env, etc).
+    # Get all lexers from a serialized array.
+    # This avoids needing to spawn mentos when it's not really needed
+    # (e.g., one-off jobs, loading the Rails env, etc).
     #
     # Should be preferred to #lexers!
     #
-    # Returns an array of lexers.
+    # @return [Array<String>] an array of lexers
     def lexers
+      lexer_file = File.join(__dir__, '..', '..', 'lexers')
       begin
-        lexer_file = File.expand_path('../../../lexers', __FILE__)
-        raw = File.open(lexer_file, "rb").read
-        Marshal.load(raw)
+        File.open(lexer_file, 'rb') do |f|
+          Marshal.load(f)
+        end
       rescue Errno::ENOENT
-        raise MentosError, "Error loading lexer file. Was it created and vendored?"
+        raise MentosError, %(Error loading #{lexer_file}. Was it created and vendored?)
       end
     end
 
-    # Public: Get back all available lexers from mentos itself
+    # Get back all available lexers from mentos itself
     #
-    # Returns an array of lexers.
+    # @return [Array<String>] an array of lexers
     def lexers!
-      mentos(:get_all_lexers).inject(Hash.new) do |hash, lxr|
+      mentos(:get_all_lexers).each_with_object({}) do |lxr, hash|
         name = lxr[0]
         hash[name] = {
-          :name => name,
-          :aliases => lxr[1],
-          :filenames => lxr[2],
-          :mimetypes => lxr[3]
+          name: name,
+          aliases: lxr[1],
+          filenames: lxr[2],
+          mimetypes: lxr[3]
         }
-        hash["dasm16"] = {:name=>"dasm16", :aliases=>["DASM16"], :filenames=>["*.dasm16", "*.dasm"], :mimetypes=>['text/x-dasm16']}
-        hash["Puppet"] = {:name=>"Puppet", :aliases=>["puppet"], :filenames=>["*.pp"], :mimetypes=>[]}
-        hash["Augeas"] = {:name=>"Augeas", :aliases=>["augeas"], :filenames=>["*.aug"], :mimetypes=>[]}
-        hash["TOML"]   = {:name=>"TOML",   :aliases=>["toml"],   :filenames=>["*.toml"], :mimetypes=>[]}
-        hash["Slash"]  = {:name=>"Slash",  :aliases=>["slash"],  :filenames=>["*.sl"], :mimetypes=>[]}
-        hash
       end
     end
 
-    # Public: Return an array of all available filters
+    # @return [Array<String>] an array of all available filters
     def filters
       mentos(:get_all_filters)
     end
 
-    # Public: Return an array of all available styles
+    # @return [Array<String>] an array of all available styles
     def styles
       mentos(:get_all_styles)
     end
 
-    # Public: Return css for highlighted code
-    def css(klass='', opts={})
+    # @return [String] css for highlighted code
+    def css(klass = '', opts = {})
       if klass.is_a?(Hash)
         opts = klass
         klass = ''
@@ -196,32 +155,29 @@ module Pygments
       mentos(:css, ['html', klass], opts)
     end
 
-    # Public: Return the name of a lexer.
+    # @return [String, nil] the name of a lexer.
     def lexer_name_for(*args)
       # Pop off the last arg if it's a hash, which becomes our opts
-      if args.last.is_a?(Hash)
-        opts = args.pop
-      else
-        opts = {}
-      end
+      opts = if args.last.is_a?(Hash)
+               args.pop
+             else
+               {}
+             end
 
-      if args.last.is_a?(String)
-        code = args.pop
-      else
-        code = nil
-      end
+      code = (args.pop if args.last.is_a?(String))
 
       mentos(:lexer_name_for, args, opts, code)
     end
 
     # Public: Highlight code.
     #
-    # Takes a first-position argument of the code to be highlighted, and a
-    # second-position hash of various arguments specifying highlighting properties.
+    # Takes a first-position argument of the code to be highlighted,
+    # and a second-position hash of various arguments specifying
+    # highlighting properties.
     #
     # Returns the highlighted string
     # or nil when the request to the Python process timed out.
-    def highlight(code, opts={})
+    def highlight(code, opts = {})
       # If the caller didn't give us any code, we have nothing to do,
       # so return right away.
       return code if code.nil? || code.empty?
@@ -240,221 +196,186 @@ module Pygments
 
     private
 
-    # Our 'rpc'-ish request to mentos. Requires a method name, and then optional
-    # args, kwargs, code.
-    def mentos(method, args=[], kwargs={}, original_code=nil)
-      # Open the pipe if necessary
-      start unless alive?
+    # @param argv [Array<String>]
+    def popen4(argv)
+      stdin, stdout, stderr, wait_thr = Open3.popen3(*argv, { close_others: true })
+      while (pid = wait_thr.pid).nil? && wait_thr.alive?
+        # wait_thr.pid is not immediately available on JRuby. Why???
+      end
+      [pid, stdin, stdout, stderr]
+    end
+
+    # Detect a suitable Python binary to use.
+    def find_python_binary
+      return %w[py python3 python].first { |py| !which(py).nil? } if Gem.win_platform?
+
+      # On non-Windows platforms, we simply rely on shebang
+      []
+    end
+
+    # Cross platform which command
+    # from http://stackoverflow.com/a/5471032/284795
+    def which(command)
+      exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+      ENV['PATH'].split(File::PATH_SEPARATOR).each do |dir|
+        exts.each do |ext|
+          path = File.join(dir, "#{command}#{ext}")
+          return path if File.executable?(path) && !File.directory?(path)
+        end
+      end
+      nil
+    end
+
+    # @param timeout [Integer]
+    # @param error_message [String]
+    # @yield
+    def with_watchdog(timeout, error_message)
+      state_mutex = Mutex.new
+      state = :alive
+      wd_cleanup = ConditionVariable.new
+
+      watchdog = if timeout.positive?
+                   Thread.new do
+                     state_mutex.synchronize do
+                       wd_cleanup.wait(state_mutex, timeout) if state != :finished
+                       if state != :finished
+                         @log.error error_message
+                         stop error_message
+                         state = :timeout
+                       end
+                     end
+                   end
+                 end
 
       begin
-        # Timeout requests that take too long.
-        # Invalid MENTOS_TIMEOUT results in just using default.
-        timeout_time = Integer(ENV["MENTOS_TIMEOUT"]) rescue 8
-
-        Timeout::timeout(timeout_time) do
-          # For sanity checking on both sides of the pipe when highlighting, we prepend and
-          # append an id.  mentos checks that these are 8 character ids and that they match.
-          # It then returns the id's back to Rubyland.
-          id = (0...8).map{65.+(rand(25)).chr}.join
-          code = add_ids(original_code, id) if original_code
-
-          # Add metadata to the header and generate it.
-          if code
-            bytesize = code.bytesize
-          else
-            bytesize = 0
+        yield
+      ensure
+        if watchdog
+          state_mutex.synchronize do
+            state = :finished if state == :alive
+            # wake up watchdog thread
+            wd_cleanup.signal
           end
+          watchdog.join
+        end
 
-          kwargs.freeze
-          kwargs = kwargs.merge("fd" => @out.to_i, "id" => id, "bytes" => bytesize)
-          out_header = MultiJson.dump(:method => method, :args => args, :kwargs => kwargs)
+        raise MentosError, error_message if state == :timeout
+      end
+    end
 
-          # Get the size of the header itself and write that.
-          bits = get_fixed_bits_from_header(out_header)
-          @in.write(bits)
+    # Our 'rpc'-ish request to mentos. Requires a method name, and then optional
+    # args, kwargs, code.
+    def mentos(method, args = [], kwargs = {}, code = nil)
+      # Open the pipe if necessary
+      start unless alive?
 
-          # mentos is now waiting for the header, and, potentially, code.
-          write_data(out_header, code)
+      # Add metadata to the header and generate it.
+      kwargs = kwargs.merge('bytes' => (code.nil? ? 0 : code.bytesize))
+      out_header = JSON.generate(method: method, args: args, kwargs: kwargs)
 
-          check_for_error
+      begin
+        timeout = get_timeout(kwargs.delete(:timeout))
+        res = with_watchdog(timeout, "Timeout on a mentos #{method} call") do
+          write_header(out_header, code)
 
           # mentos will now return data to us. First it sends the header.
-          header = get_header
+          header_len_bytes = @out.read(4)
+          if header_len_bytes.nil?
+            raise Errno::EPIPE, %(Failed to read response from Python process on a mentos #{method} call)
+          end
+
+          header_len = header_len_bytes.unpack('N')[0]
+          @log.info "Size in: #{header_len}"
+          header = @out.read(header_len)
 
           # Now handle the header, any read any more data required.
-          res = handle_header_and_return(header, id)
+          handle_header_and_return(header)
+        end
 
-          # Finally, return what we got.
-          return_result(res, method)
+        # Finally, return what we got.
+        return_result(res, method)
+      rescue Errno::EPIPE => e
+        begin
+          error_msg = @err.read
+          @log.error "Error running Python script: #{error_msg}"
+          stop "Error running Python script: #{error_msg}"
+          raise MentosError, %(#{e}: #{error_msg})
+        rescue Errno::EPIPE
+          @log.error e.to_s
+          stop e.to_s
+          raise e
         end
-      rescue Timeout::Error
-        # If we timeout, we need to clear out the pipe and start over.
-        @log.error "Timeout on a mentos #{method} call"
-        stop "Timeout on mentos #{method} call."
+      rescue StandardError => e
+        @log.error e.to_s
+        stop e.to_s
+        raise e
       end
-
-    rescue Errno::EPIPE, EOFError
-    stop "EPIPE"
-    raise MentosError, "EPIPE"
     end
 
-    def check_for_error
-      return if @err.closed?
-
-      timeout_time = 0.25 # set a very little timeout so that we do not hang the parser
+    # @param header [String]
+    # @param code [String, nil]
+    def write_header(header, code)
+      # Get the size of the header itself and write that.
+      @in.write([header.bytesize].pack('N'))
+      @log.info "Size out: #{header.bytesize}"
+
+      # mentos is now waiting for the header, and, potentially, code.
+      @in.write(header)
+      @log.info "Out header: #{header}"
+      @in.write(code) unless code.nil?
+      @in.flush
+    end
 
-      Timeout::timeout(timeout_time) do
-        error_msg = @err.read
+    # @param timeout [Integer, nil]
+    # @return [Integer]
+    def get_timeout(timeout)
+      return timeout unless timeout.nil?
 
-        unless error_msg.empty?
-          @log.error "Error running python script: #{error_msg}"
-          stop "Error running python script: #{error_msg}"
-          raise MentosError, error_msg
-        end
+      begin
+        Integer(ENV['MENTOS_TIMEOUT'])
+      rescue TypeError
+        0
       end
-    rescue Timeout::Error
-      # during the specified time no error were found
-      @err.close
     end
 
-
     # Based on the header we receive, determine if we need
     # to read more bytes, and read those bytes if necessary.
     #
-    # Then, do a sanity check with the ids.
-    #
-    # Returns a result — either highlighted text or metadata.
-    def handle_header_and_return(header, id)
-      if header
-        header = header_to_json(header)
-        bytes = header[:bytes]
-
-        # Read more bytes (the actual response body)
-        res = @out.read(bytes.to_i)
-
-        if header[:method] == "highlight"
-          # Make sure we have a result back; else consider this an error.
-          if res.nil?
-            @log.warn "No highlight result back from mentos."
-            stop "No highlight result back from mentos."
-            raise MentosError, "No highlight result back from mentos."
-          end
+    # @param header [String, nil]
+    # @return [String, nil] either highlighted text or metadata.
+    def handle_header_and_return(header)
+      raise MentosError, 'No header received back.' if header.nil?
 
-          # Remove the newline from Python
-          res = res[0..-2]
-          @log.info "Highlight in process."
-
-          # Get the id's
-          start_id = res[0..7]
-          end_id = res[-8..-1]
-
-          # Sanity check.
-          if not (start_id == id and end_id == id)
-            @log.error "ID's did not match. Aborting."
-            stop "ID's did not match. Aborting."
-            raise MentosError, "ID's did not match. Aborting."
-          else
-            # We're good. Remove the padding
-            res = res[10..-11]
-            @log.info "Highlighting complete."
-            res
-          end
-        end
-        res
-      else
-        @log.error "No header data back."
-        stop "No header data back."
-        raise MentosError, "No header received back."
-      end
-    end
+      @log.info "In header: #{header}"
+      header = header_to_json(header)
+      bytes = header[:bytes]
 
-    # With the code, prepend the id (with two spaces to avoid escaping weirdness if
-    # the following text starts with a slash (like terminal code), and append the
-    # id, with two padding also. This means we are sending over the 8 characters +
-    # code + 8 characters.
-    def add_ids(code, id)
-      (id + "  #{code}  #{id}").freeze
-    end
+      # Read more bytes (the actual response body)
+      res = @out.read(bytes.to_i)
 
-    # Write data to mentos, the Python process.
-    #
-    # Returns nothing.
-    def write_data(out_header, code=nil)
-      @in.write(out_header)
-      @log.info "Out header: #{out_header}"
-      @in.write(code) if code
-    end
-
-    # Sanity check for size (32-arity of 0's and 1's)
-    def size_check(size)
-      size_regex = /[0-1]{32}/
-      if size_regex.match(size)
-        true
-      else
-        false
+      if header[:method] == 'highlight'
+        # Make sure we have a result back; else consider this an error.
+        raise MentosError, 'No highlight result back from mentos.' if res.nil?
       end
-    end
 
-    # Read the header via the pipe.
-    #
-    # Returns a header.
-    def get_header
-      begin
-        size = @out.read(33)
-        size = size[0..-2]
-
-        # Sanity check the size
-        if not size_check(size)
-          @log.error "Size returned from mentos.py invalid."
-          stop "Size returned from mentos.py invalid."
-          raise MentosError, "Size returned from mentos.py invalid."
-        end
-
-        # Read the amount of bytes we should be expecting. We first
-        # convert the string of bits into an integer.
-        header_bytes = size.to_s.to_i(2) + 1
-        @log.info "Size in: #{size.to_s} (#{header_bytes.to_s})"
-        @out.read(header_bytes)
-      rescue
-        @log.error "Failed to get header."
-        stop "Failed to get header."
-        raise MentosError, "Failed to get header."
-      end
+      res
     end
 
-    # Return the final result for the API. Return Ruby objects for the methods that
-    # want them, text otherwise.
+    # @return Ruby objects for the methods that want them, text otherwise.
     def return_result(res, method)
-      unless method == :lexer_name_for || method == :highlight || method == :css
-        res = MultiJson.load(res, :symbolize_keys => true)
-      end
+      res = JSON.parse(res, symbolize_names: true) unless %i[lexer_name_for highlight css].include?(method)
       res = res.rstrip if res.class == String
       res
     end
 
     # Convert a text header into JSON for easy access.
+    # @param header [String]
+    # @return [JSON]
     def header_to_json(header)
-      @log.info "[In header: #{header} "
-      header = MultiJson.load(header, :symbolize_keys => true)
-
-      if header[:error]
-        # Raise this as a Ruby exception of the MentosError class.
-        # Stop so we don't leave the pipe in an inconsistent state.
-        @log.error "Failed to convert header to JSON."
-        stop header[:error]
-        raise MentosError, header[:error]
-      else
-        header
-      end
-    end
+      json = JSON.parse(header, symbolize_names: true)
+      raise MentosError, json[:error] unless json[:error].nil?
 
-    def get_fixed_bits_from_header(out_header)
-      size = out_header.bytesize
-
-      # Fixed 32 bits to represent the int. We return a string
-      # representation: e.g, "00000000000000000000000000011110"
-      Array.new(32) { |i| size[i] }.reverse!.join
+      json
     end
   end
 end
-