rygments 0.1.0

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+doc/

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in rygments.gemspec
+gemspec

data/README.markdown

@@ -0,0 +1,86 @@
+# Rygments
+
+Rygments is a Ruby wrapper for the awesome Python based Pygments syntax
+highlighter.
+
+Rygments differs from other implementations in that it embeds a full
+Python interpreter for the Pygments Python files as a Ruby C extension.
+This enables Rygments to reach a level of performance that is equal to
+invoking Pygments from a native Python application, and without the overhead forking
+a new process. This difference in performance really pays off when
+you are highlighting lots of smaller code fragments, e.g. for blog
+posts.
+
+Compared to Albino (another popular Pygments Ruby wrapper) Rygments is over 30% faster when using it to highlight all files from the Ruby 1.9.2 source code.
+
+# Installation
+
+## Pygments
+
+First, you need to make sure Pygments and the Python development headers are installed. How this can be done
+depends on your platform.
+
+### Debian and Ubuntu Linux
+
+    sudo apt-get install python-dev python-pygments
+
+### OSX (Homebrew)
+
+    brew install pip && pip install pygments
+
+The Python development headers are supplied with XCode.
+
+### OSX (Distutils)
+
+    sudo easy_install Pygments
+
+The Python development headers are supplied with XCode.
+
+## Rygments
+
+After installing Pygments you can install the Rygments gem via Rubygems:
+
+    gem install rygments
+
+and you're good to go!
+
+# Usage
+
+Usage is easy. First you have to require the Rygments gem:
+
+    require "rygments"
+
+Rygments then gives you two class methods that enable you to highlight code from wherever you want.
+
+To highlight source code from a string use:
+
+    Rygments.highlight_string(source_code, lexer, formatter)
+
+and to highlight source code from a file use:
+
+    Rygments.highlight_file(filename, lexer, formatter)
+
+Both methods return a string containing the highlighted code.
+
+The _lexer_ is a lowercase string specifying the programming language you want
+to highlight for, e.g. `ruby` or `sass`.
+
+The _formatter_ is a lowercase string specifying the kind of output you want, 
+e.g. `html` or `latex`.
+
+A full list of lexers and formatters is available on the Pygments
+website.
+
+# Compatibility
+
+Rygments has been tested on OSX Snow Leopard (10.6.6) and Ubuntu Linux 10.10. If you used it with success on a different platform then please let me know so I can extend this list.
+
+# License
+
+Copyright (c) 2011 Emil Loer <http://emilloer.com>
+
+Permission  is  hereby granted, free of charge, to any person obtaining a copy of  this  software  and  associated  documentation files  (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is  furnished to do so, subject to the following conditions:
+
+The  above  copyright  notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF  ANY  KIND, EXPRESS  OR  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE  AND  NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER  IN  AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN  THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/example/example.rb

@@ -0,0 +1,8 @@
+require "rubygems"
+require "rygments"
+
+# Highlight a string with some HTML in it
+puts Rygments.highlight_string("<div id=\"foo\">bar</div>", "html")
+
+# Highlight this file
+puts Rygments.highlight_file(__FILE__)

data/ext/extconf.rb

@@ -0,0 +1,16 @@
+require "mkmf"
+
+extension_name = "wrapper"
+
+dir_config(extension_name)
+
+$CFLAGS << " #{`python-config --cflags`.chomp} -Wno-strict-prototypes"
+$LDFLAGS << " #{`python-config --ldflags`.chomp}"
+
+# python-config adds arch options for ppc and x64 on Darwin. Ruby 1.9
+# does not like this, so we'll have to remove them.
+if RUBY_VERSION.start_with? "1.9" and RUBY_PLATFORM =~ /darwin/
+  $CFLAGS.gsub! /\s+-arch\s+\w+/, " "
+end
+
+create_makefile(extension_name)

data/ext/wrapper.c

@@ -0,0 +1,239 @@
+/* vim:set expandtab: */
+
+#include <ruby.h>
+
+/* Python.h redefines macros on Darwin, we don't want any warnings so we 
+ * undefine them here first. */
+#ifdef _DARWIN_C_SOURCE
+    #undef _XOPEN_SOURCE
+    #undef SIZEOF_TIME_T
+    #undef SIZEOF_LONG
+    #undef WORDS_BIGENDIAN
+#endif
+
+#include <Python.h>
+
+#define log(x) (printf(">> %s\n",(x)));
+
+/* Structure containing the state of the embedded interpreter */
+struct wrapper_struct {
+    PyObject *module;
+    PyObject *rygmentize;
+};
+
+/* References to our created classes and modules */
+static VALUE rygments_class = Qnil;
+static VALUE wrapper_class = Qnil;
+
+/* This function places a string at a certain index in the given tuple */
+static int put_in_tuple(PyObject *tuple, int index, const char *string) {
+    /* Don't accept NULL pointers */
+    if (string == NULL) {
+        return 1;
+    }
+    
+    /* Duplicate the string */
+    PyObject *pystr = PyString_FromString(string);
+    if (pystr == NULL) {
+        return 1;
+    }
+    
+    /* Inject it in the tuple */
+    return PyTuple_SetItem(tuple, index, pystr);
+}
+
+/* Initializes the python interpreter and loads the rygments Python 
+ * code */
+static int initialize(struct wrapper_struct *s) {
+    PyObject *module_name;
+
+    /* Initialize Python */
+    Py_InitializeEx(0);
+    
+    /* Load the rygments module */
+    module_name = PyString_FromString("rygments");
+    s->module = PyImport_Import(module_name);
+    Py_DECREF(module_name);
+
+    /* Check if module was loaded successfully */
+    if (s->module == NULL) {
+        PyErr_Print();
+        fprintf(stderr, "Failed to load the rygments python module\n");
+        return 1;
+    }
+    
+    /* Get a reference to the rygmentize function */
+    s->rygmentize = PyObject_GetAttrString(s->module, "rygmentize");
+
+    /* Check if we have a function reference */
+    if (!s->rygmentize || !PyCallable_Check(s->rygmentize)) {
+        Py_XDECREF(s->rygmentize);
+        Py_DECREF(s->module);
+
+        if (PyErr_Occurred()) {
+            PyErr_Print();
+        }
+
+        fprintf(stderr, "Cannot find the rygmentize function in the python module\n");
+        return 1;
+    }
+
+    return 0;
+}
+
+/*
+ * call-seq:
+ *   wrapper.highlight_string(string, lexer, formatter) -> string
+ *
+ * This method performs syntax highlighting on a string. Do not call this
+ * function directly. Instead, use the Rygments.highlight functions.
+ *
+ * The return value is always a string. An exception is raised whenever
+ * an error occurs.
+ */
+static VALUE wrapper_highlight_string(VALUE self, VALUE code, VALUE lexer, VALUE formatter) {
+    struct wrapper_struct *s;
+    PyObject *pArgs, *pValue;
+    int res;
+
+    /* Get the wrapper structure */
+    Data_Get_Struct(self, struct wrapper_struct, s);
+
+    /* Collect arguments in a tuple */
+    pArgs = PyTuple_New(3);
+    res = 0;
+    res += put_in_tuple(pArgs, 0, RSTRING_PTR(code));
+    res += put_in_tuple(pArgs, 1, RSTRING_PTR(lexer));
+    res += put_in_tuple(pArgs, 2, RSTRING_PTR(formatter));
+    
+    /* Check if we have bad arguments */
+    if (res) {
+        Py_DECREF(pArgs);
+        fprintf(stderr,"Bad arguments\n");
+        return Qnil;
+    }
+
+    /* Call the function */
+    pValue = PyObject_CallObject(s->rygmentize, pArgs);
+    Py_DECREF(pArgs);
+
+    /* Check if we have a valid result */
+    if (pValue == NULL) {
+        fprintf(stderr,"Call failed\n");
+        return Qnil;
+    }
+
+    /* Convert the pygmentized result to a Ruby string */
+    VALUE result = rb_str_new2(PyString_AsString(pValue));
+    Py_DECREF(pValue);
+
+    return result;
+}
+
+/*
+ * call-seq:
+ *   wrapper.highlight_file(filename, lexer, formatter) -> string
+ *
+ * This method performs syntax highlighting on a file. Do not call this
+ * function directly. Instead, use the Rygments.highlight functions.
+ *
+ * The return value is always a string. An exception is raised whenever
+ * an error occurs.
+ */
+static VALUE wrapper_highlight_file(VALUE self, VALUE filename, VALUE lexer, VALUE formatter) {
+    FILE *f;
+
+    /* Open the file */
+    if ((f=fopen(RSTRING_PTR(filename),"r"))==NULL) {
+        printf("error: could not open file\n");
+        return Qnil;
+    }
+    
+    /* Determine the file size */
+    fseek(f,0,SEEK_END);
+    long fsize=ftell(f);
+    rewind(f);
+    
+    /* Read the contents of the file into a buffer */
+    char *buf=ALLOC_N(char, fsize);
+    if ((fread(buf,1,fsize,f))!=(size_t)fsize) {
+        printf("error: could not read from file \"\"\n");
+        fclose(f);
+        return Qnil;
+    }
+    fclose(f);
+
+    /* Convert the buffer to a Ruby string */
+    VALUE code = rb_str_new(buf, fsize);
+    free(buf);
+
+    /* Call the string highlighting method */
+    return wrapper_highlight_string(self, code, lexer, formatter);
+}
+
+/* Free the wrapper structure and deinitialize the Python interpreter */
+static void wrapper_free(void *p) {
+    struct wrapper_struct *s = (struct wrapper_struct *)p;
+    
+    /* Decrese refcounts of the Python helper */
+    Py_DECREF(s->rygmentize);
+    Py_DECREF(s->module);
+
+    /* If we came here through an error we should print int */
+    if (PyErr_Occurred()) {
+        PyErr_Print();
+    }
+    
+    /* Free the Python interpreter */
+    Py_Finalize();
+
+    /* Release our structure pointer */
+    free(p);
+}
+
+/* 
+ * call-seq:
+ *   Wrapper.new(helper_path) -> Wrapper instance
+ *
+ * Creates a new instance of the Python wrapper. Do not use this
+ * function directly. Instead, use the Rygments.highlight functions
+ * which will take care of handling the wrapper.
+ *
+ * Note: The Wrapper should be treated as a singleton. It is not designed
+ * to handle multiple instances. Furthermore, on Darwin initializing,
+ * finalizing and again initializing the Python interpreter results in
+ * segmentation faults.
+ */
+static VALUE wrapper_new(VALUE klass, VALUE path) {
+    /* Patch the Python search path to include the rygments module path */
+    setenv("PYTHONPATH", RSTRING_PTR(path), 1);
+    
+    /* Initialize the Python interpreter and load the helper module */
+    struct wrapper_struct *ptr = ALLOC(struct wrapper_struct);
+    initialize(ptr);
+
+    /* Wrap the state structure */
+    VALUE tdata = Data_Wrap_Struct(klass, 0, wrapper_free, ptr);
+    return tdata;
+}
+
+/* 
+ * This Ruby C extension defines a Rygments::Wrapper class.
+ *
+ * The Rygments::Wrapper class is a wrapper around the Python interpreter.
+ * This means that Pygments can be invoked multiple times without having
+ * to fork or reload anything, thus reducing the overhead to a minimum.
+ *
+ * As a Rygments user you do not have to deal with the wrapper system
+ * directly. Instead, you should use the static functions that are exposed 
+ * in the Rygments module.
+ */
+void Init_wrapper(void) {
+    rygments_class = rb_define_module("Rygments");
+    wrapper_class = rb_define_class_under(rygments_class, "Wrapper", rb_cObject);
+
+    rb_define_singleton_method(wrapper_class, "new", wrapper_new, 1);
+
+    rb_define_method(wrapper_class, "highlight_string", wrapper_highlight_string, 3);
+    rb_define_method(wrapper_class, "highlight_file", wrapper_highlight_file, 3);
+}

data/lib/rygments/rygments.py

@@ -0,0 +1,8 @@
+from pygments import highlight
+from pygments.lexers import get_lexer_by_name
+from pygments.formatters import get_formatter_by_name
+
+def rygmentize(code, lexer, formatter):
+    lexer = get_lexer_by_name(lexer)
+    formatter = get_formatter_by_name(formatter)
+    return highlight(code, lexer, formatter)

data/lib/rygments/version.rb

@@ -0,0 +1,3 @@
+module Rygments
+  VERSION = "0.1.0"
+end

data/lib/rygments.rb

@@ -0,0 +1,65 @@
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "ext", "wrapper"))
+
+# = Introduction
+#
+# Rygments is a Ruby wrapper for the awesome Python based Pygments syntax
+# highlighter.
+#
+# Rygments differs from other implementations in that it embeds a full
+# Python interpreter for the Pygments Python files as a Ruby C extension.
+# This enables Rygments to reach a level of performance that is equal to
+# invoking Pygments from a native Python application without forking
+# a new process. This difference in performance really pays off when
+# you are highlighting lots of smaller code fragments, e.g. for a blog
+# post.
+#
+# = Usage
+#
+# Usage is easy. Rygments comes with two class methods that enable you
+# to highlight code from wherever you want.
+#
+# To highlight source code from a string use:
+#
+#   Rygments.highlight_string(source_code, lexer, formatter)
+#
+# and to highlight source code from a file use:
+#
+#   Rygments.highlight_file(filename, lexer, formatter)
+#
+# Both functions return a string containing the highlighted code.
+#
+# The _lexer_ is a string specifying the programming language you want
+# to highlight for, e.g. +ruby+ or +sass+.
+#
+# The _formatter_ is a string specifying the kind of output you want, 
+# e.g. +html+ or +latex+.
+#
+# A full list of lexers and formatters is available on the Pygments
+# website.
+#
+# Author:: Emil Loer (http://emilloer.com)
+# License:: Licensed under the terms of the MIT license
+module Rygments
+  # Perform syntax highlighting on a string.
+  # Returns the highlighted string.
+  def self.highlight_string(code, lexer = "ruby", formatter = "html")
+    wrapper.highlight_string code, lexer, formatter
+  end
+  
+  # Reads a file and performs syntax highlighting on the file's contents.
+  # Returns the highlighted string.
+  def self.highlight_file(filename, lexer = "ruby", formatter = "html")
+    wrapper.highlight_file filename, lexer, formatter
+  end
+
+  private
+
+  # The full filename of the Python helper script
+  HELPER_PATH = File.expand_path(File.join(File.dirname(__FILE__), "rygments"))
+
+  # This function maintains a single instance of the Python wrapper 
+  # extension.
+  def self.wrapper
+    @wrapper ||= Rygments::Wrapper.new(HELPER_PATH)
+  end
+end

data/rygments.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "rygments/version"
+
+Gem::Specification.new do |s|
+  s.name        = "rygments"
+  s.version     = Rygments::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Emil Loer"]
+  s.email       = ["emil@koffietijd.net"]
+  s.homepage    = "https://github.com/thedjinn/rygments"
+  s.summary     = %q{Rygments is a Ruby wrapper for Pygments}
+  s.description = %q{Rygments is a Ruby wrapper for the Pygments syntax highlighter. It uses an embedded Python interpreter to get high processing throughput.}
+
+  s.rubyforge_project = "rygments"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.has_rdoc = true
+  s.extensions = ["ext/extconf.rb"]
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification 
+name: rygments
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Emil Loer
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-02-27 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: Rygments is a Ruby wrapper for the Pygments syntax highlighter. It uses an embedded Python interpreter to get high processing throughput.
+email: 
+- emil@koffietijd.net
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.markdown
+- Rakefile
+- example/example.rb
+- ext/extconf.rb
+- ext/wrapper.c
+- lib/rygments.rb
+- lib/rygments/rygments.py
+- lib/rygments/version.rb
+- rygments.gemspec
+has_rdoc: true
+homepage: https://github.com/thedjinn/rygments
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: rygments
+rubygems_version: 1.5.2
+signing_key: 
+specification_version: 3
+summary: Rygments is a Ruby wrapper for Pygments
+test_files: []
+