pygments.rb-jruby 0.5.4

data/lib/pygments/mentos.py

@@ -0,0 +1,351 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys, re, os, signal
+import traceback
+if 'PYGMENTS_PATH' in os.environ:
+    sys.path.insert(0, os.environ['PYGMENTS_PATH'])
+
+dirname = os.path.dirname
+
+base_dir = dirname(dirname(dirname(os.path.abspath(__file__))))
+sys.path.append(base_dir + "/vendor")
+sys.path.append(base_dir + "/vendor/pygments-main")
+sys.path.append(base_dir + "/vendor/simplejson")
+
+import pygments
+from pygments import lexers, formatters, styles, filters
+
+from threading import Lock
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+def _convert_keys(dictionary):
+    if not isinstance(dictionary, dict):
+        return dictionary
+    return dict((str(k), _convert_keys(v))
+        for k, v in dictionary.items())
+
+def _write_error(error):
+    res = {"error": error}
+    out_header = json.dumps(res).encode('utf-8')
+    bits = _get_fixed_bits_from_header(out_header)
+    sys.stdout.write(bits + "\n")
+    sys.stdout.flush()
+    sys.stdout.write(out_header + "\n")
+    sys.stdout.flush()
+    return
+
+def _get_fixed_bits_from_header(out_header):
+    size = len(out_header)
+    return "".join(map(lambda y:str((size>>y)&1), range(32-1, -1, -1)))
+
+def _signal_handler(signal, frame):
+    """
+    Handle the signal given in the first argument, exiting gracefully
+    """
+    sys.exit(0)
+
+class Mentos(object):
+    """
+    Interacts with pygments.rb to provide access to pygments functionality
+    """
+    def __init__(self):
+        pass
+
+    def return_lexer(self, lexer, args, inputs, code=None):
+        """
+        Accepting a variety of possible inputs, return a Lexer object.
+
+        The inputs argument should be a hash with at least one of the following
+        keys:
+
+            - 'lexer' ("python")
+            - 'mimetype' ("text/x-ruby")
+            - 'filename' ("yeaaah.py")
+
+        The code argument should be a string, such as "import derp".
+
+        The code guessing method is not especially great. It is advised that
+        clients pass in a literal lexer name whenever possible, which provides
+        the best probability of match (100 percent).
+        """
+
+        if lexer:
+            if inputs:
+                return lexers.get_lexer_by_name(lexer, **inputs)
+            else:
+                return lexers.get_lexer_by_name(lexer)
+
+        if inputs:
+            if 'lexer' in inputs:
+                return lexers.get_lexer_by_name(inputs['lexer'], **inputs)
+
+            elif 'mimetype' in inputs:
+                return lexers.get_lexer_for_mimetype(inputs['mimetype'], **inputs)
+
+            elif 'filename' in inputs:
+                name = inputs['filename']
+
+                # If we have code and a filename, pygments allows us to guess
+                # with both. This is better than just guessing with code.
+                if code:
+                    return lexers.guess_lexer_for_filename(name, code, **inputs)
+                else:
+                    return lexers.get_lexer_for_filename(name, **inputs)
+
+        # If all we got is code, try anyway.
+        if code:
+            return lexers.guess_lexer(code, **inputs)
+
+        else:
+            return None
+
+
+    def highlight_text(self, code, lexer, formatter_name, args, kwargs):
+        """
+        Highlight the relevant code, and return a result string.
+        The default formatter is html, but alternate formatters can be passed in via
+        the formatter_name argument. Additional paramters can be passed as args
+        or kwargs.
+        """
+        # Default to html if we don't have the formatter name.
+        if formatter_name:
+            _format_name = str(formatter_name)
+        else:
+            _format_name = "html"
+
+        # Return a lexer object
+        lexer = self.return_lexer(lexer, args, kwargs, code)
+
+        # Make sure we sucessfuly got a lexer
+        if lexer:
+            formatter = pygments.formatters.get_formatter_by_name(str.lower(_format_name), **kwargs)
+
+            # Do the damn thing.
+            res = pygments.highlight(code, lexer, formatter)
+
+            return res
+
+        else:
+            _write_error("No lexer")
+
+    def get_data(self, method, lexer, args, kwargs, text=None):
+        """
+        Based on the method argument, determine the action we'd like pygments
+        to do. Then return the data generated from pygments.
+        """
+        if kwargs:
+            formatter_name = kwargs.get("formatter", None)
+            opts = kwargs.get("options", {})
+
+        # Ensure there's a 'method' key before proceeeding
+        if method:
+            res = None
+
+            # Now check what that method is. For the get methods, pygments
+            # itself returns generators, so we make them lists so we can serialize
+            # easier.
+            if method == 'get_all_styles':
+                res = json.dumps(list(pygments.styles.get_all_styles()))
+
+            elif method == 'get_all_filters':
+                res = json.dumps(list(pygments.filters.get_all_filters()))
+
+            elif method == 'get_all_lexers':
+                res = json.dumps(list(pygments.lexers.get_all_lexers()))
+
+            elif method == 'get_all_formatters':
+                res = [ [ft.__name__, ft.name, ft.aliases] for ft in pygments.formatters.get_all_formatters() ]
+                res = json.dumps(res)
+
+            elif method == 'highlight':
+                try:
+                    text = text.decode('utf-8')
+                except UnicodeDecodeError:
+                    # The text may already be encoded
+                    text = text
+                res = self.highlight_text(text, lexer, formatter_name, args, _convert_keys(opts))
+
+            elif method == 'css':
+                kwargs = _convert_keys(kwargs)
+                fmt = pygments.formatters.get_formatter_by_name(args[0], **kwargs)
+                res = fmt.get_style_defs(args[1])
+
+            elif method == 'lexer_name_for':
+                lexer = self.return_lexer(None, args, kwargs, text)
+
+                if lexer:
+                    # We don't want the Lexer itself, just the name.
+                    # Take the first alias.
+                    res = lexer.aliases[0]
+
+                else:
+                    _write_error("No lexer")
+
+            else:
+                _write_error("Invalid method " + method)
+
+            return res
+
+
+    def _send_data(self, res, method):
+
+        # Base header. We'll build on this, adding keys as necessary.
+        base_header = {"method": method}
+
+        res_bytes = len(res) + 1
+        base_header["bytes"] = res_bytes
+
+        out_header = json.dumps(base_header).encode('utf-8')
+
+        # Following the protocol, send over a fixed size represenation of the
+        # size of the JSON header
+        bits = _get_fixed_bits_from_header(out_header)
+
+        # Send it to Rubyland
+        sys.stdout.write(bits + "\n")
+        sys.stdout.flush()
+
+        # Send the header.
+        sys.stdout.write(out_header + "\n")
+        sys.stdout.flush()
+
+        # Finally, send the result
+        sys.stdout.write(res + "\n")
+        sys.stdout.flush()
+
+
+    def _get_ids(self, text):
+        start_id = text[:8]
+        end_id = text[-8:]
+        return start_id, end_id
+
+    def _check_and_return_text(self, text, start_id, end_id):
+
+        # Sanity check.
+        id_regex = re.compile('[A-Z]{8}')
+
+        if not id_regex.match(start_id) and not id_regex.match(end_id):
+            _write_error("ID check failed. Not an ID.")
+
+        if not start_id == end_id:
+            _write_error("ID check failed. ID's did not match.")
+
+        # Passed the sanity check. Remove the id's and return
+        text = text[10:-10]
+        return text
+
+    def _parse_header(self, header):
+        method = header["method"]
+        args = header.get("args", [])
+        kwargs = header.get("kwargs", {})
+        lexer = kwargs.get("lexer", None)
+        return (method, args, kwargs, lexer)
+
+    def start(self):
+        """
+        Main loop, waiting for inputs on stdin. When it gets some data,
+        it goes to work.
+
+        mentos exposes most of the "High-level API" of pygments. It always
+        expects and requires a JSON header of metadata. If there is data to be
+        pygmentized, this header will be followed by the text to be pygmentized.
+
+        The header is of form:
+        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128, "fd": "8"}
+        """
+        lock = Lock()
+
+        while True:
+            # The loop begins by reading off a simple 32-arity string
+            # representing an integer of 32 bits. This is the length of
+            # our JSON header.
+            size = sys.stdin.read(32)
+
+            lock.acquire()
+
+            try:
+                # Read from stdin the amount of bytes we were told to expect.
+                header_bytes = int(size, 2)
+
+                # Sanity check the size
+                size_regex = re.compile('[0-1]{32}')
+                if not size_regex.match(size):
+                    _write_error("Size received is not valid.")
+
+                line = sys.stdin.read(header_bytes)
+
+                header = json.loads(line)
+
+                method, args, kwargs, lexer = self._parse_header(header)
+                _bytes = 0
+
+                if lexer:
+                    lexer = str(lexer)
+
+                # Read more bytes if necessary
+                if kwargs:
+                    _bytes = kwargs.get("bytes", 0)
+
+                # Read up to the given number bytes (possibly 0)
+                text = sys.stdin.read(_bytes)
+
+                # Sanity check the return.
+                if _bytes:
+                    start_id, end_id = self._get_ids(text)
+                    text = self._check_and_return_text(text, start_id, end_id)
+
+                # Get the actual data from pygments.
+                res = self.get_data(method, lexer, args, kwargs, text)
+
+                # Put back the sanity check values.
+                if method == "highlight":
+                    res = start_id + "  " + res + "  " + end_id
+
+                self._send_data(res, method)
+
+            except:
+                tb = traceback.format_exc()
+                _write_error(tb)
+
+            finally:
+                lock.release()
+
+def main():
+
+    # Signal handlers to trap signals.
+    signal.signal(signal.SIGINT, _signal_handler)
+    signal.signal(signal.SIGTERM, _signal_handler)
+    if sys.platform != "win32":
+        signal.signal(signal.SIGHUP, _signal_handler)
+
+    mentos = Mentos()
+
+    if sys.platform == "win32":
+        # disable CRLF
+        import msvcrt
+        msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+    else:
+        # close fd's inherited from the ruby parent
+        import resource
+        maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+        if maxfd == resource.RLIM_INFINITY:
+            maxfd = 65536
+
+        for fd in range(3, maxfd):
+            try:
+                os.close(fd)
+            except:
+                pass
+
+    mentos.start()
+
+if __name__ == "__main__":
+    main()
+
+
+

data/lib/pygments/popen.rb

@@ -0,0 +1,404 @@
+# coding: utf-8
+require 'yajl'
+require 'timeout'
+require 'logger'
+require 'time'
+
+# Error class
+class MentosError < IOError
+end
+
+# Pygments provides access to the Pygments library via a pipe and a long-running
+# Python process.
+module Pygments
+  module Popen
+    extend self
+
+    # Get things started by opening a pipe to mentos (the freshmaker), a
+    # Python process that talks to the Pygments library. We'll talk back and
+    # forth across this pipe.
+    def start(pygments_path = File.expand_path('../../../vendor/pygments-main/', __FILE__))
+      is_windows = RUBY_PLATFORM =~ /mswin|mingw/
+      begin
+        @log = Logger.new(ENV['MENTOS_LOG'] ||= is_windows ? 'NUL:' : '/dev/null')
+        @log.level = Logger::INFO
+        @log.datetime_format = "%Y-%m-%d %H:%M "
+      rescue
+        @log = Logger.new(is_windows ? 'NUL:' : '/dev/null')
+      end
+
+      ENV['PYGMENTS_PATH'] = pygments_path
+
+      # Make sure we kill off the child when we're done
+      at_exit { stop "Exiting" }
+
+      # A pipe to the mentos python process. #popen4 gives us
+      # the pid and three IO objects to write and read.
+      script = "#{python_binary} #{File.expand_path('../mentos.py', __FILE__)}"
+      @pid, @in, @out, @err = popen4(script)
+      @log.info "[#{Time.now.iso8601}] Starting pid #{@pid.to_s} with fd #{@out.to_i.to_s}."
+    end
+
+    # Detect a suitable Python binary to use. We can't just use `python2`
+    # because apparently some old versions of Debian only have `python` or
+    # something like that.
+    def python_binary
+      @python_binary ||= begin
+        `which python2`
+        $?.success? ? "python2" : "python"
+      end
+    end
+
+    # Stop the child process by issuing a kill -9.
+    #
+    # We then call waitpid() with the pid, which waits for that particular
+    # child and reaps it.
+    #
+    # kill() can set errno to ESRCH if, for some reason, the file
+    # is gone; regardless the final outcome of this method
+    # will be to set our @pid variable to nil.
+    #
+    # Technically, kill() can also fail with EPERM or EINVAL (wherein
+    # the signal isn't sent); but we have permissions, and
+    # we're not doing anything invalid here.
+    def stop(reason)
+      if @pid
+        begin
+          Process.kill('KILL', @pid)
+          Process.waitpid(@pid)
+        rescue Errno::ESRCH, Errno::ECHILD
+        end
+      end
+      @log.info "[#{Time.now.iso8601}] Killing pid: #{@pid.to_s}. Reason: #{reason}"
+      @pid = nil
+    end
+
+    # Check for a @pid variable, and then hit `kill -0` with the pid to
+    # check if the pid is still in the process table. If this function
+    # gives us an ENOENT or ESRCH, we can also safely return false (no process
+    # to worry about). Defensively, if EPERM is raised, in a odd/rare
+    # dying process situation (e.g., mentos is checking on the pid of a dead
+    # process and the pid has already been re-used) we'll want to raise
+    # that as a more informative Mentos exception.
+    #
+    # Returns true if the child is alive.
+    def alive?
+      return true if @pid && Process.kill(0, @pid)
+      false
+    rescue Errno::ENOENT, Errno::ESRCH
+      false
+    rescue Errno::EPERM
+      raise MentosError, "EPERM checking if child process is alive."
+    end
+
+    # Public: Get an array of available Pygments formatters
+    #
+    # Returns an array of formatters.
+    def formatters
+      mentos(:get_all_formatters).inject(Hash.new) do | hash, (name, desc, aliases) |
+        # Remove the long-winded and repetitive 'Formatter' suffix
+        name.sub!(/Formatter$/, '')
+        hash[name] = {
+          :name => name,
+          :description => desc,
+          :aliases => aliases
+        }
+        hash
+      end
+    end
+
+
+    # Public: Get all lexers from a serialized array. This avoids needing to spawn
+    # mentos when it's not really needed (e.g,. one-off jobs, loading the Rails env, etc).
+    #
+    # Should be preferred to #lexers!
+    #
+    # Returns an array of lexers
+    def lexers
+      begin
+        lexer_file = File.expand_path('../../../lexers', __FILE__)
+        raw = File.open(lexer_file, "r").read
+        Marshal.load(raw)
+      rescue Errno::ENOENT
+        raise MentosError, "Error loading lexer file. Was it created and vendored?"
+      end
+    end
+
+    # Public: Get back all available lexers from mentos itself
+    #
+    # Returns an array of lexers
+    def lexers!
+      mentos(:get_all_lexers).inject(Hash.new) do |hash, lxr|
+        name = lxr[0]
+        hash[name] = {
+          :name => name,
+          :aliases => lxr[1],
+          :filenames => lxr[2],
+          :mimetypes => lxr[3]
+        }
+        hash["Augeas"] = {:name=>"Augeas", :aliases=>["augeas"], :filenames=>["*.aug"], :mimetypes=>[]}
+        hash["dasm16"] = {:name=>"dasm16", :aliases=>["DASM16"], :filenames=>["*.dasm16", "*.dasm"], :mimetypes=>['text/x-dasm16']}
+        hash["Puppet"] = {:name=>"Puppet", :aliases=>["puppet"], :filenames=>["*.pp"], :mimetypes=>[]}
+        hash["Slash"]  = {:name=>"Slash",  :aliases=>["slash"],  :filenames=>["*.sl"], :mimetypes=>[]}
+        hash
+      end
+    end
+
+    # Public: Return an array of all available filters
+    def filters
+      mentos(:get_all_filters)
+    end
+
+    # Public: Return an array of all available styles
+    def styles
+      mentos(:get_all_styles)
+    end
+
+    # Public: Return css for highlighted code
+    def css(klass='', opts={})
+      if klass.is_a?(Hash)
+        opts = klass
+        klass = ''
+      end
+      mentos(:css, ['html', klass], opts)
+    end
+
+    # Public: Return the name of a lexer.
+    def lexer_name_for(*args)
+      # Pop off the last arg if it's a hash, which becomes our opts
+      if args.last.is_a?(Hash)
+        opts = args.pop
+      else
+        opts = {}
+      end
+
+      if args.last.is_a?(String)
+        code = args.pop
+      else
+        code = nil
+      end
+
+      mentos(:lexer_name_for, args, opts, code)
+    end
+
+    # Public: Highlight code.
+    #
+    # Takes a first-position argument of the code to be highlighted, and a
+    # second-position hash of various arguments specifiying highlighting properties.
+    def highlight(code, opts={})
+      # If the caller didn't give us any code, we have nothing to do,
+      # so return right away.
+      return code if code.nil? || code.empty?
+
+      # Callers pass along options in the hash
+      opts[:options] ||= {}
+
+      # Default to utf-8 for the output encoding, if not given.
+      opts[:options][:outencoding] ||= 'utf-8'
+
+      # Get back the string from mentos and force encoding if we can
+      str = mentos(:highlight, nil, opts, code)
+      str.force_encoding(opts[:options][:outencoding]) if str.respond_to?(:force_encoding)
+      str
+    end
+
+    private
+
+    # Our 'rpc'-ish request to mentos. Requires a method name, and then optional
+    # args, kwargs, code.
+    def mentos(method, args=[], kwargs={}, original_code=nil)
+      # Open the pipe if necessary
+      start unless alive?
+
+      begin
+        # Timeout requests that take too long.
+        timeout_time = 8
+
+        Timeout::timeout(timeout_time) do
+          # For sanity checking on both sides of the pipe when highlighting, we prepend and
+          # append an id.  mentos checks that these are 8 character ids and that they match.
+          # It then returns the id's back to Rubyland.
+          id = (0...8).map{65.+(rand(25)).chr}.join
+          code = add_ids(original_code, id) if original_code
+
+          # Add metadata to the header and generate it.
+          if code
+            bytesize = code.bytesize
+          else
+            bytesize = 0
+          end
+
+          kwargs.freeze
+          kwargs = kwargs.merge("fd" => @out.to_i, "id" => id, "bytes" => bytesize)
+          out_header = Yajl.dump(:method => method, :args => args, :kwargs => kwargs)
+
+          # Get the size of the header itself and write that.
+          bits = get_fixed_bits_from_header(out_header)
+          @in.write(bits)
+
+          # mentos is now waiting for the header, and, potentially, code.
+          write_data(out_header, code)
+
+          # mentos will now return data to us. First it sends the header.
+          header = get_header
+
+          # Now handle the header, any read any more data required.
+          res = handle_header_and_return(header, id)
+
+          # Finally, return what we got.
+          return_result(res, method)
+        end
+      rescue Timeout::Error
+        # If we timeout, we need to clear out the pipe and start over.
+        @log.error "[#{Time.now.iso8601}] Timeout on a mentos #{method} call"
+        stop "Timeout on mentos #{method} call."
+      end
+
+    rescue Errno::EPIPE, EOFError
+    stop "EPIPE"
+    raise MentosError, "EPIPE"
+    end
+
+
+    # Based on the header we receive, determine if we need
+    # to read more bytes, and read those bytes if necessary.
+    #
+    # Then, do a sanity check wih the ids.
+    #
+    # Returns a result — either highlighted text or metadata.
+    def handle_header_and_return(header, id)
+      if header
+        header = header_to_json(header)
+        bytes = header["bytes"]
+
+        # Read more bytes (the actual response body)
+        res = @out.read(bytes.to_i)
+
+        if header["method"] == "highlight"
+          # Make sure we have a result back; else consider this an error.
+          if res.nil?
+            @log.warn "[#{Time.now.iso8601}] No highlight result back from mentos."
+            stop "No highlight result back from mentos."
+            raise MentosError, "No highlight result back from mentos."
+          end
+
+          # Remove the newline from Python
+          res = res[0..-2]
+          @log.info "[#{Time.now.iso8601}] Highlight in process."
+
+          # Get the id's
+          start_id = res[0..7]
+          end_id = res[-8..-1]
+
+          # Sanity check.
+          if not (start_id == id and end_id == id)
+            @log.error "[#{Time.now.iso8601}] ID's did not match. Aborting."
+            stop "ID's did not match. Aborting."
+            raise MentosError, "ID's did not match. Aborting."
+          else
+            # We're good. Remove the padding
+            res = res[10..-11]
+            @log.info "[#{Time.now.iso8601}] Highlighting complete."
+            res
+          end
+        end
+        res
+      else
+        @log.error "[#{Time.now.iso8601}] No header data back."
+        stop "No header data back."
+        raise MentosError, "No header received back."
+      end
+    end
+
+    # With the code, prepend the id (with two spaces to avoid escaping weirdness if
+    # the following text starts with a slash (like terminal code), and append the
+    # id, with two padding also. This means we are sending over the 8 characters +
+    # code + 8 characters.
+    def add_ids(code, id)
+      code.freeze
+      code = id + "  #{code}  #{id}"
+      code
+    end
+
+    # Write data to mentos, the Python Process.
+    #
+    # Returns nothing.
+    def write_data(out_header, code=nil)
+      @in.write(out_header)
+      @log.info "[#{Time.now.iso8601}] Out header: #{out_header.to_s}"
+      @in.write(code) if code
+    end
+
+    # Sanity check for size (32-arity of 0's and 1's)
+    def size_check(size)
+      size_regex = /[0-1]{32}/
+      if size_regex.match(size)
+        true
+      else
+        false
+      end
+    end
+
+    # Read the header via the pipe.
+    #
+    # Returns a header.
+    def get_header
+      begin
+        size = @out.read(33)
+        size = size[0..-2]
+
+        # Sanity check the size
+        if not size_check(size)
+          @log.error "[#{Time.now.iso8601}] Size returned from mentos.py invalid."
+          stop "Size returned from mentos.py invalid."
+          raise MentosError, "Size returned from mentos.py invalid."
+        end
+
+        # Read the amount of bytes we should be expecting. We first
+        # convert the string of bits into an integer.
+        header_bytes = size.to_s.to_i(2) + 1
+        @log.info "[#{Time.now.iso8601}] Size in: #{size.to_s} (#{header_bytes.to_s})"
+        @out.read(header_bytes)
+      rescue
+        @log.error "[#{Time.now.iso8601}] Failed to get header."
+        stop "Failed to get header."
+        raise MentosError, "Failed to get header."
+      end
+    end
+
+    # Return the final result for the API. Return Ruby objects for the methods that
+    # want them, text otherwise.
+    def return_result(res, method)
+      unless method == :lexer_name_for || method == :highlight || method == :css
+        res = Yajl.load(res, :symbolize_keys => true)
+      end
+      res = res.rstrip if res.class == String
+      res
+    end
+
+    # Convert a text header into JSON for easy access.
+    def header_to_json(header)
+      @log.info "[#{Time.now.iso8601}] In header: #{header.to_s} "
+      header = Yajl.load(header)
+
+      if header["error"]
+        # Raise this as a Ruby exception of the MentosError class.
+        # Stop so we don't leave the pipe in an inconsistent state.
+        @log.error "[#{Time.now.iso8601}] Failed to convert header to JSON."
+        stop header["error"]
+        raise MentosError, header["error"]
+      else
+        header
+      end
+    end
+
+    def get_fixed_bits_from_header(out_header)
+      size = out_header.bytesize
+
+      # Fixed 32 bits to represent the int. We return a string
+      # represenation: e.g, "00000000000000000000000000011110"
+      Array.new(32) { |i| size[i] }.reverse!.join
+    end
+  end
+end
+