pygments.rb 1.2.1 → 2.0.0

data/lib/pygments/mentos.py

@@ -1,6 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+import struct
 import sys, re, os, signal
 import traceback
 if 'PYGMENTS_PATH' in os.environ:
@@ -9,15 +10,11 @@ if 'PYGMENTS_PATH' in os.environ:
 dirname = os.path.dirname
 
 base_dir = dirname(dirname(dirname(os.path.abspath(__file__))))
-sys.path.append(base_dir + "/vendor")
 sys.path.append(base_dir + "/vendor/pygments-main")
-sys.path.append(base_dir + "/vendor/simplejson")
 
 import pygments
 from pygments import lexers, formatters, styles, filters
 
-from threading import Lock
-
 try:
     import json
 except ImportError:
@@ -27,22 +24,16 @@ def _convert_keys(dictionary):
     if not isinstance(dictionary, dict):
         return dictionary
     return dict((str(k), _convert_keys(v))
-        for k, v in dictionary.items())
+        for k, v in list(dictionary.items()))
 
 def _write_error(error):
     res = {"error": error}
-    out_header = json.dumps(res).encode('utf-8')
-    bits = _get_fixed_bits_from_header(out_header)
-    sys.stdout.write(bits + "\n")
-    sys.stdout.flush()
-    sys.stdout.write(out_header + "\n")
+    out_header_bytes = json.dumps(res).encode('utf-8')
+    sys.stdout.buffer.write(struct.pack('!i', len(out_header_bytes)))
+    sys.stdout.buffer.write(out_header_bytes)
     sys.stdout.flush()
     return
 
-def _get_fixed_bits_from_header(out_header):
-    size = len(out_header)
-    return "".join(map(lambda y:str((size>>y)&1), range(32-1, -1, -1)))
-
 def _signal_handler(signal, frame):
     """
     Handle the signal given in the first argument, exiting gracefully
@@ -163,12 +154,9 @@ class Mentos(object):
                 res = json.dumps(res)
 
             elif method == 'highlight':
-                try:
-                    text = text.decode('utf-8')
-                except UnicodeDecodeError:
-                    # The text may already be encoded
-                    text = text
                 res = self.highlight_text(text, lexer, formatter_name, args, _convert_keys(opts))
+                if type(res) is bytes:
+                    res = res.decode('utf-8')
 
             elif method == 'css':
                 kwargs = _convert_keys(kwargs)
@@ -193,52 +181,21 @@ class Mentos(object):
 
 
     def _send_data(self, res, method):
-
         # Base header. We'll build on this, adding keys as necessary.
         base_header = {"method": method}
 
-        res_bytes = len(res) + 1
-        base_header["bytes"] = res_bytes
+        res_bytes = res.encode("utf-8")
+        bytes = len(res_bytes)
+        base_header["bytes"] = bytes
 
-        out_header = json.dumps(base_header).encode('utf-8')
-
-        # Following the protocol, send over a fixed size represenation of the
-        # size of the JSON header
-        bits = _get_fixed_bits_from_header(out_header)
+        out_header_bytes = json.dumps(base_header).encode('utf-8')
 
         # Send it to Rubyland
-        sys.stdout.write(bits + "\n")
-        sys.stdout.flush()
-
-        # Send the header.
-        sys.stdout.write(out_header + "\n")
+        sys.stdout.buffer.write(struct.pack('!i', len(out_header_bytes)))
+        sys.stdout.buffer.write(out_header_bytes)
+        sys.stdout.buffer.write(res_bytes)
         sys.stdout.flush()
 
-        # Finally, send the result
-        sys.stdout.write(res + "\n")
-        sys.stdout.flush()
-
-
-    def _get_ids(self, text):
-        start_id = text[:8]
-        end_id = text[-8:]
-        return start_id, end_id
-
-    def _check_and_return_text(self, text, start_id, end_id):
-
-        # Sanity check.
-        id_regex = re.compile('[A-Z]{8}')
-
-        if not id_regex.match(start_id) and not id_regex.match(end_id):
-            _write_error("ID check failed. Not an ID.")
-
-        if not start_id == end_id:
-            _write_error("ID check failed. ID's did not match.")
-
-        # Passed the sanity check. Remove the id's and return
-        text = text[10:-10]
-        return text
-
     def _parse_header(self, header):
         method = header["method"]
         args = header.get("args", [])
@@ -256,29 +213,18 @@ class Mentos(object):
         pygmentized, this header will be followed by the text to be pygmentized.
 
         The header is of form:
-        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128, "fd": "8"}
+        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128}
         """
-        lock = Lock()
 
         while True:
-            # The loop begins by reading off a simple 32-arity string
-            # representing an integer of 32 bits. This is the length of
-            # our JSON header.
-            size = sys.stdin.read(32)
+            header_size_bytes = sys.stdin.buffer.read(4)
+            if not header_size_bytes:
+                break
 
-            lock.acquire()
+            header_size = struct.unpack('!i', header_size_bytes)[0]
 
             try:
-                # Read from stdin the amount of bytes we were told to expect.
-                header_bytes = int(size, 2)
-
-                # Sanity check the size
-                size_regex = re.compile('[0-1]{32}')
-                if not size_regex.match(size):
-                    _write_error("Size received is not valid.")
-
-                line = sys.stdin.read(header_bytes)
-
+                line = sys.stdin.buffer.read(header_size).decode('utf-8')
                 header = json.loads(line)
 
                 method, args, kwargs, lexer = self._parse_header(header)
@@ -291,30 +237,18 @@ class Mentos(object):
                 if kwargs:
                     _bytes = kwargs.get("bytes", 0)
 
-                # Read up to the given number bytes (possibly 0)
-                text = sys.stdin.read(_bytes)
-
-                # Sanity check the return.
-                if _bytes:
-                    start_id, end_id = self._get_ids(text)
-                    text = self._check_and_return_text(text, start_id, end_id)
+                # Read up to the given number of *bytes* (not chars) (possibly 0)
+                text = sys.stdin.buffer.read(_bytes).decode('utf-8')
 
                 # Get the actual data from pygments.
                 res = self.get_data(method, lexer, args, kwargs, text)
 
-                # Put back the sanity check values.
-                if method == "highlight":
-                    res = start_id + "  " + res + "  " + end_id
-
                 self._send_data(res, method)
 
             except:
                 tb = traceback.format_exc()
                 _write_error(tb)
 
-            finally:
-                lock.release()
-
 def main():
 
     # Signal handlers to trap signals.
@@ -324,28 +258,7 @@ def main():
         signal.signal(signal.SIGHUP, _signal_handler)
 
     mentos = Mentos()
-
-    if sys.platform == "win32":
-        # disable CRLF
-        import msvcrt
-        msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
-    else:
-        # close fd's inherited from the ruby parent
-        import resource
-        maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
-        if maxfd == resource.RLIM_INFINITY:
-            maxfd = 65536
-
-        for fd in range(3, maxfd):
-            try:
-                os.close(fd)
-            except:
-                pass
-
     mentos.start()
 
 if __name__ == "__main__":
     main()
-
-
-