rubyjedi-oga 1.0.3

data/ext/java/org/liboga/xml/Lexer.rl

@@ -0,0 +1,223 @@
+package org.liboga.xml;
+
+%%machine java_lexer;
+
+import java.io.IOException;
+
+import org.jcodings.Encoding;
+
+import org.jruby.Ruby;
+import org.jruby.RubyModule;
+import org.jruby.RubyClass;
+import org.jruby.RubyObject;
+import org.jruby.RubyString;
+import org.jruby.RubyFixnum;
+import org.jruby.util.ByteList;
+import org.jruby.anno.JRubyClass;
+import org.jruby.anno.JRubyMethod;
+import org.jruby.runtime.ThreadContext;
+import org.jruby.runtime.ObjectAllocator;
+import org.jruby.runtime.builtin.IRubyObject;
+
+/**
+ * Lexer support class for JRuby.
+ *
+ * The Lexer class contains the raw Ragel loop and calls back in to Ruby land
+ * whenever a Ragel action is needed similar to the C extension setup.
+ *
+ * This class requires Ruby land to first define the `Oga::XML` namespace.
+ */
+@JRubyClass(name="Oga::XML::Lexer", parent="Object")
+public class Lexer extends RubyObject
+{
+    /**
+     * The current Ruby runtime.
+     */
+    private Ruby runtime;
+
+    %% write data;
+
+    /* Used by Ragel to keep track of the current state. */
+    int act;
+    int cs;
+    int top;
+    int lines;
+    int[] stack;
+
+    /**
+     * Sets up the current class in the Ruby runtime.
+     */
+    public static void load(Ruby runtime)
+    {
+        RubyModule xml = (RubyModule) runtime.getModule("Oga")
+            .getConstant("XML");
+
+        RubyClass lexer = xml.defineClassUnder(
+            "Lexer",
+            runtime.getObject(),
+            ALLOCATOR
+        );
+
+        lexer.defineAnnotatedMethods(Lexer.class);
+    }
+
+    private static final ObjectAllocator ALLOCATOR = new ObjectAllocator()
+    {
+        public IRubyObject allocate(Ruby runtime, RubyClass klass)
+        {
+            return new org.liboga.xml.Lexer(runtime, klass);
+        }
+    };
+
+    public Lexer(Ruby runtime, RubyClass klass)
+    {
+        super(runtime, klass);
+
+        this.runtime = runtime;
+    }
+
+    /**
+     * Runs the bulk of the Ragel loop and calls back in to Ruby.
+     *
+     * This method pulls its data in from the instance variable `@data`. The
+     * Ruby side of the Lexer class should set this variable to a String in its
+     * constructor method. Encodings are passed along to make sure that token
+     * values share the same encoding as the input.
+     *
+     * This method always returns nil.
+     */
+    @JRubyMethod
+    public IRubyObject advance_native(ThreadContext context, RubyString rb_str)
+    {
+        Boolean html_p = this.callMethod(context, "html?").isTrue();
+
+        Encoding encoding = rb_str.getEncoding();
+
+        byte[] data = rb_str.getBytes();
+
+        int ts    = 0;
+        int te    = 0;
+        int p     = 0;
+        int mark  = 0;
+        int lines = this.lines;
+        int pe    = data.length;
+        int eof   = data.length;
+
+        String id_advance_line        = "advance_line";
+        String id_on_attribute        = "on_attribute";
+        String id_on_attribute_ns     = "on_attribute_ns";
+        String id_on_cdata_start      = "on_cdata_start";
+        String id_on_cdata_body       = "on_cdata_body";
+        String id_on_cdata_end        = "on_cdata_end";
+        String id_on_comment_start    = "on_comment_start";
+        String id_on_comment_body     = "on_comment_body";
+        String id_on_comment_end      = "on_comment_end";
+        String id_on_doctype_end      = "on_doctype_end";
+        String id_on_doctype_inline   = "on_doctype_inline";
+        String id_on_doctype_name     = "on_doctype_name";
+        String id_on_doctype_start    = "on_doctype_start";
+        String id_on_doctype_type     = "on_doctype_type";
+        String id_on_element_end      = "on_element_end";
+        String id_on_element_name     = "on_element_name";
+        String id_on_element_ns       = "on_element_ns";
+        String id_on_element_open_end = "on_element_open_end";
+        String id_on_proc_ins_end     = "on_proc_ins_end";
+        String id_on_proc_ins_name    = "on_proc_ins_name";
+        String id_on_proc_ins_start   = "on_proc_ins_start";
+        String id_on_proc_ins_body    = "on_proc_ins_body";
+        String id_on_string_body      = "on_string_body";
+        String id_on_string_dquote    = "on_string_dquote";
+        String id_on_string_squote    = "on_string_squote";
+        String id_on_text             = "on_text";
+        String id_on_xml_decl_end     = "on_xml_decl_end";
+        String id_on_xml_decl_start   = "on_xml_decl_start";
+
+        %% write exec;
+
+        this.lines = lines;
+
+        return context.nil;
+    }
+
+    /**
+     * Resets the internal state of the lexer.
+     */
+    @JRubyMethod
+    public IRubyObject reset_native(ThreadContext context)
+    {
+        this.act   = 0;
+        this.top   = 0;
+        this.stack = new int[4];
+        this.cs    = java_lexer_start;
+
+        return context.nil;
+    }
+
+    /**
+     * Calls back in to Ruby land passing the current token value along.
+     *
+     * This method calls back in to Ruby land based on the method name
+     * specified in `name`. The Ruby callback should take one argument. This
+     * argument will be a String containing the value of the current token.
+     */
+    public void callback(String name, byte[] data, Encoding enc, int ts, int te)
+    {
+        ByteList bytelist = new ByteList(data, ts, te - ts, enc, true);
+
+        RubyString value = this.runtime.newString(bytelist);
+
+        ThreadContext context = this.runtime.getCurrentContext();
+
+        this.callMethod(context, name, value);
+    }
+
+    /**
+     * Calls back in to Ruby land without passing any arguments.
+     */
+    public void callback_simple(String name)
+    {
+        ThreadContext context = this.runtime.getCurrentContext();
+
+        this.callMethod(context, name);
+    }
+
+    /**
+     * Advances the line number by `amount` lines.
+     */
+    public void advance_line(int amount)
+    {
+        ThreadContext context = this.runtime.getCurrentContext();
+        RubyFixnum lines      = this.runtime.newFixnum(amount);
+
+        this.callMethod(context, "advance_line", lines);
+    }
+
+    /**
+     * @see Oga::XML::Lexer#html_script?
+     */
+    public Boolean html_script_p()
+    {
+        ThreadContext context = this.runtime.getCurrentContext();
+
+        return this.callMethod(context, "html_script?").isTrue();
+    }
+
+    /**
+     * @see Oga::XML::Lexer#html_style?
+     */
+    public Boolean html_style_p()
+    {
+        ThreadContext context = this.runtime.getCurrentContext();
+
+        return this.callMethod(context, "html_style?").isTrue();
+    }
+}
+
+%%{
+    variable act this.act;
+    variable cs this.cs;
+    variable stack this.stack;
+    variable top this.top;
+
+    include base_lexer "base_lexer.rl";
+}%%

data/ext/ragel/base_lexer.rl

@@ -0,0 +1,633 @@
+%%machine base_lexer;
+
+%%{
+    ##
+    # Base grammar for the XML lexer.
+    #
+    # This grammar is shared between the C and Java extensions. As a result of
+    # this you should **not** include language specific code in Ragel
+    # actions/callbacks.
+    #
+    # To call back in to Ruby you can use one of the following two functions:
+    #
+    # * callback
+    # * callback_simple
+    #
+    # The first function takes 5 arguments:
+    #
+    # * The name of the Ruby method to call.
+    # * The input data.
+    # * The encoding of the input data.
+    # * The start of the current buffer.
+    # * The end of the current buffer.
+    #
+    # The function callback_simple only takes one argument: the name of the
+    # method to call. This function should be used for callbacks that don't
+    # require any values.
+    #
+    # When you call a method in Ruby make sure that said method is defined as
+    # an instance method in the `Oga::XML::Lexer` class.
+    #
+    # The name of the callback to invoke should be an identifier starting with
+    # "id_". The identifier should be defined in the associated C and Java code.
+    # In case of C code its value should be a Symbol as a ID object, for Java
+    # it should be a String. For example:
+    #
+    #     ID id_foo = rb_intern("foo");
+    #
+    # And for Java:
+    #
+    #     String id_foo = "foo";
+    #
+    # ## Machine Transitions
+    #
+    # To transition from one machine to another always use `fnext` instead of
+    # `fcall` and `fret`. This removes the need for the code to keep track of a
+    # stack.
+    #
+
+    newline    = '\r\n' | '\n' | '\r';
+    whitespace = [ \t];
+    ident_char = [a-zA-Z0-9\-_\.];
+    identifier = ident_char+;
+
+    whitespace_or_newline = whitespace | newline;
+
+    action count_newlines {
+        if ( fc == '\n' ) lines++;
+    }
+
+    action advance_newline {
+        advance_line(1);
+    }
+
+    action hold_and_return {
+        fhold;
+        fret;
+    }
+
+    # Comments
+    #
+    # http://www.w3.org/TR/html/syntax.html#comments
+    #
+    # Unlike the W3C specification these rules *do* allow character sequences
+    # such as `--` and `->`. Putting extra checks in for these sequences would
+    # actually make the rules/actions more complex.
+    #
+
+    comment_start = '<!--';
+    comment_end   = '-->';
+
+    # Everything except "-" OR a single "-"
+    comment_allowed = (^'-'+ | '-') $count_newlines;
+
+    action start_comment {
+        callback_simple(id_on_comment_start);
+
+        fnext comment_body;
+    }
+
+    comment_body := |*
+        comment_allowed => {
+            callback(id_on_comment_body, data, encoding, ts, te);
+
+            if ( lines > 0 )
+            {
+                advance_line(lines);
+
+                lines = 0;
+            }
+        };
+
+        comment_end => {
+            callback_simple(id_on_comment_end);
+
+            fnext main;
+        };
+    *|;
+
+    # CDATA
+    #
+    # http://www.w3.org/TR/html/syntax.html#cdata-sections
+    #
+    # In HTML CDATA tags have no meaning/are not supported. Oga does
+    # support them but treats their contents as plain text.
+    #
+
+    cdata_start = '<![CDATA[';
+    cdata_end   = ']]>';
+
+    # Everything except "]" OR a single "]"
+    cdata_allowed = (^']'+ | ']') $count_newlines;
+
+    action start_cdata {
+        callback_simple(id_on_cdata_start);
+
+        fnext cdata_body;
+    }
+
+    cdata_body := |*
+        cdata_allowed => {
+            callback(id_on_cdata_body, data, encoding, ts, te);
+
+            if ( lines > 0 )
+            {
+                advance_line(lines);
+
+                lines = 0;
+            }
+        };
+
+        cdata_end => {
+            callback_simple(id_on_cdata_end);
+
+            fnext main;
+        };
+    *|;
+
+    # Processing Instructions
+    #
+    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
+    # http://en.wikipedia.org/wiki/Processing_Instruction
+    #
+    # These are tags meant to be used by parsers/libraries for custom behaviour.
+    # One example are the tags used by PHP: <?php and ?>. Note that the XML
+    # declaration tags (<?xml ?>) are not considered to be a processing
+    # instruction.
+    #
+
+    proc_ins_start = '<?' identifier;
+    proc_ins_end   = '?>';
+
+    # Everything except "?" OR a single "?"
+    proc_ins_allowed = (^'?'+ | '?') $count_newlines;
+
+    action start_proc_ins {
+        callback_simple(id_on_proc_ins_start);
+        callback(id_on_proc_ins_name, data, encoding, ts + 2, te);
+
+        fnext proc_ins_body;
+    }
+
+    proc_ins_body := |*
+        proc_ins_allowed => {
+            callback(id_on_proc_ins_body, data, encoding, ts, te);
+
+            if ( lines > 0 )
+            {
+                advance_line(lines);
+
+                lines = 0;
+            }
+        };
+
+        proc_ins_end => {
+            callback_simple(id_on_proc_ins_end);
+
+            fnext main;
+        };
+    *|;
+
+    # Strings
+    #
+    # Strings in HTML can either be single or double quoted. If a string
+    # starts with one of these quotes it must be closed with the same type
+    # of quote.
+    #
+    dquote = '"';
+    squote = "'";
+
+    action emit_string {
+        callback(id_on_string_body, data, encoding, ts, te);
+
+        if ( lines > 0 )
+        {
+            advance_line(lines);
+
+            lines = 0;
+        }
+    }
+
+    action start_string_squote {
+        callback_simple(id_on_string_squote);
+
+        fcall string_squote;
+    }
+
+    action start_string_dquote {
+        callback_simple(id_on_string_dquote);
+
+        fcall string_dquote;
+    }
+
+    string_squote := |*
+        ^squote* $count_newlines => emit_string;
+
+        squote => {
+            callback_simple(id_on_string_squote);
+
+            fret;
+        };
+    *|;
+
+    string_dquote := |*
+        ^dquote* $count_newlines => emit_string;
+
+        dquote => {
+            callback_simple(id_on_string_dquote);
+
+            fret;
+        };
+    *|;
+
+    # DOCTYPES
+    #
+    # http://www.w3.org/TR/html/syntax.html#the-doctype
+    #
+    # These rules support the 3 flavours of doctypes:
+    #
+    # 1. Normal doctypes, as introduced in the HTML5 specification.
+    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
+    # 3. Legacy doctypes
+    #
+    doctype_start = '<!DOCTYPE'i (whitespace_or_newline+ $count_newlines);
+
+    action start_doctype {
+        callback_simple(id_on_doctype_start);
+
+        if ( lines > 0 )
+        {
+            advance_line(lines);
+
+            lines = 0;
+        }
+
+        fnext doctype;
+    }
+
+    # Machine for processing inline rules of a doctype.
+    doctype_inline := |*
+        ^']'* $count_newlines => {
+            callback(id_on_doctype_inline, data, encoding, ts, te);
+
+            if ( lines > 0 )
+            {
+                advance_line(lines);
+
+                lines = 0;
+            }
+        };
+
+        ']' => { fnext doctype; };
+    *|;
+
+    # Machine for processing doctypes. Doctype values such as the public
+    # and system IDs are treated as T_STRING tokens.
+    doctype := |*
+        'PUBLIC' | 'SYSTEM' => {
+            callback(id_on_doctype_type, data, encoding, ts, te);
+        };
+
+        # Starts a set of inline doctype rules.
+        '[' => { fnext doctype_inline; };
+
+        # Lex the public/system IDs as regular strings.
+        squote => start_string_squote;
+        dquote => start_string_dquote;
+
+        identifier => {
+            callback(id_on_doctype_name, data, encoding, ts, te);
+        };
+
+        '>' => {
+            callback_simple(id_on_doctype_end);
+            fnext main;
+        };
+
+        newline => advance_newline;
+
+        whitespace;
+    *|;
+
+    # XML declaration tags
+    #
+    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
+    #
+    xml_decl_start = '<?xml';
+    xml_decl_end   = '?>';
+
+    action start_xml_decl {
+        callback_simple(id_on_xml_decl_start);
+        fnext xml_decl;
+    }
+
+    # Machine that processes the contents of an XML declaration tag.
+    xml_decl := |*
+        xml_decl_end => {
+            if ( lines > 0 )
+            {
+                advance_line(lines);
+
+                lines = 0;
+            }
+
+            callback_simple(id_on_xml_decl_end);
+
+            fnext main;
+        };
+
+        # Attributes and their values (e.g. version="1.0").
+        identifier => {
+            if ( lines > 0 )
+            {
+                advance_line(lines);
+
+                lines = 0;
+            }
+
+            callback(id_on_attribute, data, encoding, ts, te);
+        };
+
+        squote => start_string_squote;
+        dquote => start_string_dquote;
+
+        any $count_newlines;
+    *|;
+
+    # Elements
+    #
+    # http://www.w3.org/TR/html/syntax.html#syntax-elements
+    #
+    # Lexing of elements is broken up into different machines that handle the
+    # name/namespace, contents of the open tag and the body of an element. The
+    # body of an element is lexed using the `main` machine.
+    #
+
+    action start_element {
+        fhold;
+        fnext element_name;
+    }
+
+    action start_close_element {
+        fnext element_close;
+    }
+
+    action close_element {
+        callback(id_on_element_end, data, encoding, ts, te);
+    }
+
+    action close_element_fnext_main {
+        callback_simple(id_on_element_end);
+
+        fnext main;
+    }
+
+    element_start = '<' ident_char;
+    element_end   = '</';
+
+    # Machine used for lexing the name/namespace of an element.
+    element_name := |*
+        identifier ':' => {
+            callback(id_on_element_ns, data, encoding, ts, te - 1);
+        };
+
+        identifier => {
+            callback(id_on_element_name, data, encoding, ts, te);
+            fnext element_head;
+        };
+    *|;
+
+    # Machine used for lexing the closing tag of an element
+    element_close := |*
+        # namespace prefixes, currently not used but allows the rule below it
+        # to be used for the actual element name.
+        identifier ':';
+
+        identifier => close_element;
+
+        '>' => {
+            if ( lines > 0 )
+            {
+                advance_line(lines);
+
+                lines = 0;
+            }
+
+            fnext main;
+        };
+
+        any $count_newlines;
+    *|;
+
+    # Characters that can be used for unquoted HTML attribute values.
+    # See https://html.spec.whatwg.org/multipage/introduction.html#intro-early-example
+    # for more info.
+    html_unquoted_value =
+        ^(squote | dquote | whitespace_or_newline)
+        ^('`' | '=' | '<' | '>' | whitespace_or_newline)+;
+
+    # Machine used after matching the "=" of an attribute and just before moving
+    # into the actual attribute value.
+    attribute_pre := |*
+        whitespace_or_newline $count_newlines;
+
+        any => {
+            fhold;
+
+            if ( lines > 0 )
+            {
+                advance_line(lines);
+
+                lines = 0;
+            }
+
+            if ( html_p )
+            {
+                fnext html_attribute_value;
+            }
+            else
+            {
+                fnext xml_attribute_value;
+            }
+        };
+    *|;
+
+    # Machine used for processing HTML attribute values.
+    html_attribute_value := |*
+        squote | dquote => {
+            fhold;
+            fnext xml_attribute_value;
+        };
+
+        # Unquoted attribute values are lexed as if they were single quoted
+        # strings.
+        html_unquoted_value => {
+            callback_simple(id_on_string_squote);
+
+            callback(id_on_string_body, data, encoding, ts, te);
+
+            callback_simple(id_on_string_squote);
+        };
+
+        any => hold_and_return;
+    *|;
+
+    # Machine used for processing XML attribute values.
+    xml_attribute_value := |*
+        # The following two actions use "fnext" instead of "fcall". Combined
+        # with "element_head" using "fcall" to jump to this machine this means
+        # we can return back to "element_head" after processing a single string.
+        squote => {
+            callback_simple(id_on_string_squote);
+
+            fnext string_squote;
+        };
+
+        dquote => {
+            callback_simple(id_on_string_dquote);
+
+            fnext string_dquote;
+        };
+
+        any => hold_and_return;
+    *|;
+
+    # Machine used for processing the contents of an element's starting tag.
+    # This includes the name, namespace and attributes.
+    element_head := |*
+        newline => advance_newline;
+
+        # Attribute names and namespaces.
+        identifier ':' => {
+            callback(id_on_attribute_ns, data, encoding, ts, te - 1);
+        };
+
+        identifier => {
+            callback(id_on_attribute, data, encoding, ts, te);
+        };
+
+        # Attribute values.
+        '=' => {
+            fcall attribute_pre;
+        };
+
+        # We're done with the open tag of the element.
+        '>' => {
+            callback_simple(id_on_element_open_end);
+
+            if ( html_script_p() )
+            {
+                fnext html_script;
+            }
+            else if ( html_style_p() )
+            {
+                fnext html_style;
+            }
+            else
+            {
+                fnext main;
+            }
+        };
+
+        # Self closing tags.
+        '/>' => {
+            callback_simple(id_on_element_end);
+            fnext main;
+        };
+
+        any;
+    *|;
+
+    # Text
+    #
+    # http://www.w3.org/TR/xml/#syntax
+    # http://www.w3.org/TR/html/syntax.html#text
+    #
+    # Text content is everything leading up to certain special tags such as "</"
+    # and "<?".
+
+    action start_text {
+        fhold;
+        fnext text;
+    }
+
+    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
+    # back to the main machine.
+    #
+    # Note that this only works if each sequence is exactly 2 characters
+    # long. Because of this "<!" is used instead of "<!--".
+
+    terminate_text = '</' | '<!' | '<?' | element_start;
+    allowed_text   = (any* -- terminate_text) $count_newlines;
+
+    action emit_text {
+        callback(id_on_text, data, encoding, ts, te);
+
+        if ( lines > 0 )
+        {
+            advance_line(lines);
+
+            lines = 0;
+        }
+    }
+
+    text := |*
+        terminate_text | allowed_text => {
+            callback(id_on_text, data, encoding, ts, te);
+
+            if ( lines > 0 )
+            {
+                advance_line(lines);
+
+                lines = 0;
+            }
+
+            fnext main;
+        };
+
+        # Text followed by a special tag, such as "foo<!--"
+        allowed_text %{ mark = p; } terminate_text => {
+            callback(id_on_text, data, encoding, ts, mark);
+
+            p    = mark - 1;
+            mark = 0;
+
+            if ( lines > 0 )
+            {
+                advance_line(lines);
+
+                lines = 0;
+            }
+
+            fnext main;
+        };
+    *|;
+
+    # Certain tags in HTML can contain basically anything except for the literal
+    # closing tag. Two examples are script and style tags.  As a result of this
+    # we can't use the regular text machine.
+
+    literal_html_allowed = (^'<'+ | '<'+) $count_newlines;
+
+    html_script := |*
+        literal_html_allowed => emit_text;
+        '</script>'          => close_element_fnext_main;
+    *|;
+
+    html_style := |*
+        literal_html_allowed => emit_text;
+        '</style>'           => close_element_fnext_main;
+    *|;
+
+    # The main machine aka the entry point of Ragel.
+    main := |*
+        doctype_start  => start_doctype;
+        xml_decl_start => start_xml_decl;
+        comment_start  => start_comment;
+        cdata_start    => start_cdata;
+        proc_ins_start => start_proc_ins;
+        element_start  => start_element;
+        element_end    => start_close_element;
+        any            => start_text;
+    *|;
+}%%