pygments.rb 0.2.13 → 0.3.0

data/.gitignore

@@ -3,3 +3,4 @@ ext/Makefile
 lib/pygments_ext.*
 tmp
 pkg
+*.pyc

data/README.md

@@ -1,14 +1,23 @@
 # pygments.rb
 
-A ruby wrapper for the python [pygments syntax highlighter](http://pygments.org/).
+A Ruby wrapper for the Python [pygments syntax highlighter](http://pygments.org/).
 
-This library replaces [github/albino](https://github.com/github/albino).
-Instead of shelling out to `pygmentize`, it embeds the python
-interpreter inside ruby via FFI. This avoids the cost of setting up the
-python VM on every invocation and speeds up code highlighting from ruby by 10-15x.
+pygments.rb works by talking over a simple pipe to a long-lived 
+Python child process. This library replaces [github/albino](https://github.com/github/albino),
+as well as a version of pygments.rb that used an embedded Python
+interpreter. 
+
+Each Ruby process that runs has its own 'personal Python'; 
+for example, 4 Unicorn workers will have one Python process each. 
+If a Python process dies, a new one will be spawned on the next 
+pygments.rb request. 
 
 ## usage
 
+``` ruby 
+require 'pygments'
+``` 
+
 ``` ruby
 Pygments.highlight(File.read(__FILE__), :lexer => 'ruby')
 ```
@@ -20,29 +29,34 @@ options hash:
 Pygments.highlight('code', :options => {:encoding => 'utf-8'})
 ```
 
-To use a formatter other than html, specify it explicitly:
+pygments.rb defaults to using an HTML formatter. 
+To use a formatter other than `html`, specify it explicitly
+like so:
 
 ``` ruby
 Pygments.highlight('code', :formatter => 'bbcode')
 Pygments.highlight('code', :formatter => 'terminal')
 ```
 
-To generate CSS for html formatted code, use the css method:
+To generate CSS for HTML formatted code, use the `#css` method:
 
 ``` ruby
 Pygments.css
 Pygments.css('.highlight')
 ```
 
-To use a custom python installation (like in ArchLinux), tell
-RubyPython where python lives:
+Other Pygments high-level API methods are also available.
+These methods return arrays detailing all the available lexers, formatters, 
+and styles.
 
 ``` ruby
-RubyPython.configure :python_exe => 'python2.7'
+Pygments.lexers
+Pygments.formatters
+Pygments.styles
 ```
 
 To use a custom pygments installation, specify the path to
-Pygments.start:
+`Pygments#start`:
 
 ``` ruby
 Pygments.start("/path/to/pygments")
@@ -50,12 +64,24 @@ Pygments.start("/path/to/pygments")
 
 ## benchmarks
 
-    $ ruby -rubygems bench.rb 50
-                                   user     system      total        real
-    albino                     0.050000   0.050000  12.830000 ( 13.180806)
-    pygments::c                1.000000   0.010000   1.010000 (  1.009348)
-    pygments::ffi + reload    11.350000   1.240000  12.590000 ( 12.692320)
-    pygments::ffi              1.130000   0.010000   1.140000 (  1.171589)
 
-To run `bench.rb`, use a git checkout. The C extension is not included
-in gem releases.
+    $ ruby bench.rb 50
+       Benchmarking....
+       Size: 698 bytes
+       Iterations: 50
+                                                      user     system      total        real
+       pygments popen                                0.010000   0.010000   0.020000 (  0.460370)
+       pygments popen (process already started)      0.010000   0.000000   0.010000 (  0.272975)
+       pygments popen (process already started 2)    0.000000   0.000000   0.000000 (  0.273589)
+
+    $ ruby bench.rb 10
+       Benchmarking....
+       Size: 15523 bytes
+       Iterations: 10
+                                                      user     system      total        real
+       pygments popen                               0.000000   0.000000   0.000000 (  0.819419)
+       pygments popen (process already started)     0.010000   0.000000   0.010000 (  0.676515)
+       pygments popen (process already started 2)   0.000000   0.010000   0.010000 (  0.674189)
+
+
+

data/Rakefile

@@ -1,3 +1,6 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
 task :default => :test
 
 # ==========================================================
@@ -10,16 +13,6 @@ require 'rake/gempackagetask'
 Rake::GemPackageTask.new(GEMSPEC) do |pkg|
 end
 
-# ==========================================================
-# Ruby Extension
-# ==========================================================
-
-require 'rake/extensiontask'
-Rake::ExtensionTask.new('pygments_ext', GEMSPEC) do |ext|
-  ext.ext_dir = 'ext'
-end
-task :build => :compile
-
 # ==========================================================
 # Testing
 # ==========================================================
@@ -29,7 +22,24 @@ Rake::TestTask.new 'test' do |t|
   t.test_files = FileList['test/test_*.rb']
   t.ruby_opts = ['-rubygems']
 end
-task :test => :build
+
+# ==========================================================
+# Benchmarking
+# ==========================================================
+
+task :bench do
+  sh "ruby bench.rb"
+end
+
+# ==========================================================
+# Cache lexers
+# # ==========================================================
+
+# Write all the lexers to a file for easy lookup
+task :lexers do
+  sh "ruby cache-lexers.rb"
+end
+
 
 # ==========================================================
 # Vendor

data/bench.rb

@@ -1,55 +1,22 @@
-$:.unshift('lib')
-
+require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
 require 'benchmark'
-require 'pygments/c'
-require 'pygments/ffi'
-require 'rubygems'
-require 'albino'
 
-num = ARGV[0] ? ARGV[0].to_i : 25
-code = File.read(__FILE__)
+include Benchmark
+# number of iterations
+num = ARGV[0] ? ARGV[0].to_i : 10
 
-albino, pygments, ffi =
-  Albino.new(code, :ruby, :html).colorize,
-  Pygments::C.highlight(code, :lexer => 'ruby'),
-  Pygments::FFI.highlight(code, :lexer => 'ruby')
+# we can also repeat the code itself
+repeats = ARGV[1] ? ARGV[1].to_i : 1
 
-unless albino == pygments and pygments == ffi
-  raise "incompatible implementations (#{albino.size} != #{pygments.size} != #{ffi.size})"
-end
+code = File.open('test/test_data.py').read.to_s * repeats
 
-Benchmark.bm(25) do |x|
-  x.report('albino') do
-    num.times do
-      Albino.new(code, :ruby, :html).colorize
-    end
-  end
-  x.report('pygments::c') do
-    num.times do
-      Pygments::C.highlight(code, :lexer => 'ruby')
-    end
-  end
-  x.report('pygments::ffi + reload') do
-    num.times do
-      Pygments::FFI.start
-      Pygments::FFI.highlight(code, :lexer => 'ruby')
-      Pygments::FFI.stop
-    end
-  end
-  Pygments::FFI.start
-  x.report('pygments::ffi') do
-    num.times do
-      Pygments::FFI.highlight(code, :lexer => 'ruby')
-    end
-  end
-end
+puts "Benchmarking....\n"
+puts "Size: " + code.bytesize.to_s + " bytes\n"
+puts "Iterations: " + num.to_s + "\n"
 
-__END__
-
-$ ruby -rubygems bench.rb 50
-                               user     system      total        real
-albino                     0.050000   0.050000  12.830000 ( 13.180806)
-pygments::c                1.000000   0.010000   1.010000 (  1.009348)
-pygments::ffi + reload    11.350000   1.240000  12.590000 ( 12.692320)
-pygments::ffi              1.130000   0.010000   1.140000 (  1.171589)
+Benchmark.bm(40) do |x|
+  x.report("pygments popen                             ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+  x.report("pygments popen (process already started)   ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+  x.report("pygments popen (process already started 2) ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+end
 

data/cache-lexers.rb

@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
+
+# Simple marshalling
+serialized_lexers = Marshal.dump(Pygments.lexers!)
+
+# Write to a file
+File.open("lexers", 'w') { |file| file.write(serialized_lexers) }
+

data/lib/pygments.rb

@@ -1,11 +1,8 @@
-# require 'pygments/c'
-require 'pygments/ffi'
+require File.join(File.dirname(__FILE__), 'pygments/popen')
+
 
 module Pygments
-  # include Pygments::C
-  include Pygments::FFI
+  extend Pygments::Popen
 
   autoload :Lexer, 'pygments/lexer'
-
-  extend self
 end

data/lib/pygments/mentos.py

@@ -0,0 +1,343 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import sys, re, os, signal, resource
+import traceback
+if 'PYGMENTS_PATH' in os.environ:
+    sys.path.insert(0, os.environ['PYGMENTS_PATH'])
+
+dirname = os.path.dirname
+
+base_dir = dirname(dirname(dirname(os.path.abspath(__file__))))
+sys.path.append(base_dir + "/vendor")
+sys.path.append(base_dir + "/vendor/pygments-main")
+sys.path.append(base_dir + "/vendor/simplejson")
+
+import pygments
+from pygments import lexers, formatters, styles, filters
+
+from threading import Lock
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+def _convert_keys(dictionary):
+    if not isinstance(dictionary, dict):
+        return dictionary
+    return dict((str(k), _convert_keys(v))
+        for k, v in dictionary.items())
+
+def _write_error(error):
+    res = {"error": error}
+    out_header = json.dumps(res).encode('utf-8')
+    bits = _get_fixed_bits_from_header(out_header)
+    sys.stdout.write(bits + "\n")
+    sys.stdout.flush()
+    sys.stdout.write(out_header + "\n")
+    sys.stdout.flush()
+    return
+
+def _get_fixed_bits_from_header(out_header):
+    size = len(out_header)
+    return "".join(map(lambda y:str((size>>y)&1), range(32-1, -1, -1)))
+
+def _signal_handler(signal, frame):
+    """
+    Handle the signal given in the first argument, exiting gracefully
+    """
+    sys.exit(0)
+
+class Mentos(object):
+    """
+    Interacts with pygments.rb to provide access to pygments functionality
+    """
+    def __init__(self):
+        pass
+
+    def return_lexer(self, lexer, args, inputs, code=None):
+        """
+        Accepting a variety of possible inputs, return a Lexer object.
+
+        The inputs argument should be a hash with at least one of the following
+        keys:
+
+            - 'lexer' ("python")
+            - 'mimetype' ("text/x-ruby")
+            - 'filename' ("yeaaah.py")
+            - 'code' ("import derp" etc)
+
+        The code guessing method is not especially great. It is advised that
+        clients pass in a literal lexer name whenever possible, which provides
+        the best probability of match (100 percent).
+        """
+
+        if lexer:
+            if inputs:
+                return lexers.get_lexer_by_name(lexer, **inputs)
+            else:
+                return lexers.get_lexer_by_name(lexer)
+
+        if inputs:
+            if 'lexer' in inputs:
+                return lexers.get_lexer_by_name(inputs['lexer'], **inputs)
+
+            elif 'mimetype' in inputs:
+                return lexers.get_lexer_for_mimetype(inputs['mimetype'], **inputs)
+
+            elif 'filename' in inputs:
+                name = inputs['filename']
+
+                # If we have code and a filename, pygments allows us to guess
+                # with both. This is better than just guessing with code.
+                if code:
+                    return lexers.guess_lexer_for_filename(name, code, **inputs)
+                else:
+                    return lexers.get_lexer_for_filename(name, **inputs)
+
+        # If all we got is code, try anyway.
+        if code:
+            return lexers.guess_lexer(code, **inputs)
+
+        else:
+            _write_error("No lexer")
+
+
+    def highlight_text(self, code, lexer, formatter_name, args, kwargs):
+        """
+        Highlight the relevant code, and return a result string.
+        The default formatter is html, but alternate formatters can be passed in via
+        the formatter_name argument. Additional paramters can be passed as args
+        or kwargs.
+        """
+        # Default to html if we don't have the formatter name.
+        if formatter_name:
+            _format_name = str(formatter_name)
+        else:
+            _format_name = "html"
+
+        # Return a lexer object
+        lexer = self.return_lexer(lexer, args, kwargs, code)
+
+        # Make sure we sucessfuly got a lexer
+        if lexer:
+            formatter = pygments.formatters.get_formatter_by_name(str.lower(_format_name), **kwargs)
+
+            # Do the damn thing.
+            res = pygments.highlight(code, lexer, formatter)
+
+            return res
+
+        else:
+            _write_error("No lexer")
+
+    def get_data(self, method, lexer, args, kwargs, text=None):
+        """
+        Based on the method argument, determine the action we'd like pygments
+        to do. Then return the data generated from pygments.
+        """
+        if kwargs:
+            formatter_name = kwargs.get("formatter", None)
+            opts = kwargs.get("options", {})
+
+        # Ensure there's a 'method' key before proceeeding
+        if method:
+            res = None
+
+            # Now check what that method is. For the get methods, pygments
+            # itself returns generators, so we make them lists so we can serialize
+            # easier.
+            if method == 'get_all_styles':
+                res = json.dumps(list(pygments.styles.get_all_styles()))
+
+            elif method == 'get_all_filters':
+                res = json.dumps(list(pygments.filters.get_all_filters()))
+
+            elif method == 'get_all_lexers':
+                res = json.dumps(list(pygments.lexers.get_all_lexers()))
+
+            elif method == 'get_all_formatters':
+                res = [ [ft.__name__, ft.name, ft.aliases] for ft in pygments.formatters.get_all_formatters() ]
+                res = json.dumps(res)
+
+            elif method == 'highlight':
+                try:
+                    text = text.decode('utf-8')
+                except UnicodeDecodeError:
+                    # The text may already be encoded
+                    text = text
+                res = self.highlight_text(text, lexer, formatter_name, args, _convert_keys(opts))
+
+            elif method == 'css':
+                kwargs = _convert_keys(kwargs)
+                fmt = pygments.formatters.get_formatter_by_name(args[0], **kwargs)
+                res = fmt.get_style_defs(args[1])
+
+            elif method == 'lexer_name_for':
+                lexer = self.return_lexer(None, args, kwargs, text)
+
+                if lexer:
+                    # We don't want the Lexer itself, just the name.
+                    # Take the first alias.
+                    res = lexer.aliases[0]
+
+                else:
+                    _write_error("No lexer")
+
+            else:
+                _write_error("No lexer")
+
+            return res
+
+
+    def _send_data(self, res, method):
+
+        # Base header. We'll build on this, adding keys as necessary.
+        base_header = {"method": method}
+
+        res_bytes = len(res) + 1
+        base_header["bytes"] = res_bytes
+
+        out_header = json.dumps(base_header).encode('utf-8')
+
+        # Following the protocol, send over a fixed size represenation of the
+        # size of the JSON header
+        bits = _get_fixed_bits_from_header(out_header)
+
+        # Send it to Rubyland
+        sys.stdout.write(bits + "\n")
+        sys.stdout.flush()
+
+        # Send the header.
+        sys.stdout.write(out_header + "\n")
+        sys.stdout.flush()
+
+        # Finally, send the result
+        sys.stdout.write(res + "\n")
+        sys.stdout.flush()
+
+
+    def _get_ids(self, text):
+        start_id = text[:8]
+        end_id = text[-8:]
+        return start_id, end_id
+
+    def _check_and_return_text(self, text, start_id, end_id):
+
+        # Sanity check.
+        id_regex = re.compile('[A-Z]{8}')
+
+        if not id_regex.match(start_id) and not id_regex.match(end_id):
+            _write_error("ID check failed. Not an ID.")
+
+        if not start_id == end_id:
+            _write_error("ID check failed. ID's did not match.")
+
+        # Passed the sanity check. Remove the id's and return
+        text = text[10:-10]
+        return text
+
+    def _parse_header(self, header):
+        method = header["method"]
+        args = header.get("args", [])
+        kwargs = header.get("kwargs", {})
+        lexer = kwargs.get("lexer", None)
+        return (method, args, kwargs, lexer)
+
+    def start(self):
+        """
+        Main loop, waiting for inputs on stdin. When it gets some data,
+        it goes to work.
+
+        mentos exposes most of the "High-level API" of pygments. It always
+        expects and requires a JSON header of metadata. If there is data to be
+        pygmentized, this header will be followed by the text to be pygmentized.
+
+        The header is of form:
+        { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128, "fd": "8"}
+        """
+        lock = Lock()
+
+        while True:
+            # The loop begins by reading off a simple 32-arity string
+            # representing an integer of 32 bits. This is the length of
+            # our JSON header.
+            size = sys.stdin.read(32)
+
+            lock.acquire()
+
+            try:
+                # Read from stdin the amount of bytes we were told to expect.
+                header_bytes = int(size, 2)
+
+                # Sanity check the size
+                size_regex = re.compile('[0-1]{32}')
+                if not size_regex.match(size):
+                    _write_error("Size received is not valid.")
+
+                line = sys.stdin.read(header_bytes)
+
+                header = json.loads(line)
+
+                method, args, kwargs, lexer = self._parse_header(header)
+                _bytes = 0
+
+                if lexer:
+                    lexer = str(lexer)
+
+                # Read more bytes if necessary
+                if kwargs:
+                    _bytes = kwargs.get("bytes", 0)
+
+                # Read up to the given number bytes (possibly 0)
+                text = sys.stdin.read(_bytes)
+
+                # Sanity check the return.
+                if method == 'highlight':
+                    start_id, end_id = self._get_ids(text)
+                    text = self._check_and_return_text(text, start_id, end_id)
+
+                # Get the actual data from pygments.
+                res = self.get_data(method, lexer, args, kwargs, text)
+
+                # Put back the sanity check values.
+                if method == "highlight":
+                    res = start_id + "  " + res + "  " + end_id
+
+                self._send_data(res, method)
+
+            except:
+                tb = traceback.format_exc()
+                _write_error(tb)
+
+            finally:
+                lock.release()
+
+def main():
+
+    # Signal handlers to trap signals.
+    signal.signal(signal.SIGINT, _signal_handler)
+    signal.signal(signal.SIGTERM, _signal_handler)
+    signal.signal(signal.SIGHUP, _signal_handler)
+
+    mentos = Mentos()
+
+    # close fd's inherited from the ruby parent
+    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+    if maxfd == resource.RLIM_INFINITY:
+        maxfd = 65536
+
+    for fd in range(3, maxfd):
+        try:
+            os.close(fd)
+        except:
+            pass
+
+    mentos.start()
+
+if __name__ == "__main__":
+    main()
+
+
+