wikitext 4.0.1 → 4.0.2

data/ext/wikitext/parser.h

@@ -0,0 +1,40 @@
+// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+#include "ruby_compat.h"
+
+VALUE Wikitext_parser_initialize(int argc, VALUE *argv, VALUE self);
+
+VALUE Wikitext_parser_tokenize(VALUE self, VALUE string);
+
+VALUE Wikitext_parser_benchmarking_tokenize(VALUE self, VALUE string);
+
+VALUE Wikitext_parser_fulltext_tokenize(int argc, VALUE *argv, VALUE self);
+
+VALUE Wikitext_parser_sanitize_link_target(VALUE self, VALUE string);
+
+VALUE Wikitext_parser_encode_link_target(VALUE self, VALUE in);
+
+VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self);
+
+VALUE Wikitext_parser_profiling_parse(VALUE self, VALUE string);

data/ext/wikitext/ruby_compat.h

@@ -0,0 +1,34 @@
+// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+#include <ruby.h>
+
+// for compatibility with Ruby 1.8.5, which doesn't declare RSTRING_PTR
+#ifndef RSTRING_PTR
+#define RSTRING_PTR(s) (RSTRING(s)->ptr)
+#endif
+
+// for compatibility with Ruby 1.8.5, which doesn't declare RSTRING_LEN
+#ifndef RSTRING_LEN
+#define RSTRING_LEN(s) (RSTRING(s)->len)
+#endif

data/ext/wikitext/str.c

@@ -0,0 +1,109 @@
+// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+#include "str.h"
+
+// when allocating memory, reserve a little more than was asked for,
+// which can help to avoid subsequent allocations
+#define STR_OVERALLOC 256
+
+str_t *str_new(void)
+{
+    str_t *str      = ALLOC_N(str_t, 1);
+    str->ptr        = NULL;
+    str->len        = 0;
+    str->capacity   = 0;
+    return str;
+}
+
+str_t *str_new_copy(const char *src, long len)
+{
+    str_t *str      = ALLOC_N(str_t, 1);
+    str->ptr        = ALLOC_N(char, len + STR_OVERALLOC);
+    memcpy(str->ptr, src, len);
+    str->len        = len;
+    str->capacity   = len + STR_OVERALLOC;
+    return str;
+}
+
+str_t *str_new_from_string(VALUE string)
+{
+    string = StringValue(string);
+    return str_new_copy(RSTRING_PTR(string), RSTRING_LEN(string));
+}
+
+VALUE string_from_str(str_t *str)
+{
+    VALUE string = rb_str_new(str->ptr, str->len);
+    rb_funcall(string, rb_intern("force_encoding"), 1, rb_str_new2("UTF-8"));
+    return string;
+}
+
+void str_grow(str_t *str, long len)
+{
+    if (str->capacity < len)
+    {
+        if (str->ptr)
+            REALLOC_N(str->ptr, char, len + STR_OVERALLOC);
+        else
+            str->ptr = ALLOC_N(char, len + STR_OVERALLOC);
+        str->capacity = len + STR_OVERALLOC;
+    }
+}
+
+void str_append(str_t *str, const char *src, long len)
+{
+    long new_len = str->len + len;
+    if (str->capacity < new_len)
+    {
+        if (str->ptr)
+            REALLOC_N(str->ptr, char, new_len + STR_OVERALLOC);
+        else
+            str->ptr = ALLOC_N(char, new_len + STR_OVERALLOC);
+        str->capacity = new_len + STR_OVERALLOC;
+    }
+    memcpy(str->ptr + str->len, src, len);
+    str->len = new_len;
+}
+
+void str_append_str(str_t *str, str_t *other)
+{
+    str_append(str, other->ptr, other->len);
+}
+
+void str_append_string(str_t *str, VALUE other)
+{
+    str_append(str, RSTRING_PTR(other), RSTRING_LEN(other));
+}
+
+void str_clear(str_t *str)
+{
+    str->len = 0;
+}
+
+void str_free(str_t *str)
+{
+    if (str->ptr)
+        free(str->ptr);
+    free(str);
+}

data/ext/wikitext/str.h

@@ -0,0 +1,64 @@
+// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+#include "ruby_compat.h"
+
+typedef struct
+{
+    char *ptr;
+    long len;
+    long capacity;
+} str_t;
+
+// Mark the str struct designated by ptr as a participant in Ruby's mark-and-sweep garbage collection scheme.
+// A variable named name is placed on the C stack to prevent the structure from being prematurely collected.
+#define GC_WRAP_STR(ptr, name) volatile VALUE name __attribute__((unused)) = Data_Wrap_Struct(rb_cObject, 0, str_free, ptr)
+
+// create a new, empty string struct
+str_t *str_new(void);
+
+// create a new string struct and initialize it with a copy of the buffer of length len pointed to by src
+str_t *str_new_copy(const char *src, long len);
+
+// convenience method for testing
+str_t *str_new_from_string(VALUE string);
+
+// convenience method for testing
+VALUE string_from_str(str_t *str);
+
+// grows a string's capacity to the specified length
+void str_grow(str_t *str, long len);
+
+void str_append(str_t *str, const char *src, long len);
+
+// appends the "other" string struct onto str
+void str_append_str(str_t *str, str_t *other);
+
+// appends the "other" string (a Ruby String) onto str
+void str_append_string(str_t *str, VALUE other);
+
+// don't actually free the memory yet
+// this makes str structs very useful when reusing buffers because it avoids reallocation
+void str_clear(str_t *str);
+
+void str_free(str_t *str);

data/ext/wikitext/token.c

@@ -0,0 +1,125 @@
+// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+#include "token.h"
+#include "wikitext.h"
+
+// return a hash of token types
+// we make this available for unit testing purposes
+
+VALUE Wikitext_parser_token_types(VALUE self)
+{
+    VALUE hash = rb_hash_new();
+
+#define SET_TOKEN_TYPE(identifier)  (void)rb_hash_aset(hash, INT2FIX(identifier), \
+    rb_funcall(rb_funcall(rb_str_new2(#identifier), rb_intern("downcase"), 0), rb_intern("to_sym"), 0))
+
+    SET_TOKEN_TYPE(NO_TOKEN);
+    SET_TOKEN_TYPE(P);
+    SET_TOKEN_TYPE(LI);
+    SET_TOKEN_TYPE(NESTED_LIST);
+    SET_TOKEN_TYPE(PRE);
+    SET_TOKEN_TYPE(PRE_START);
+    SET_TOKEN_TYPE(PRE_END);
+    SET_TOKEN_TYPE(NO_WIKI_START);
+    SET_TOKEN_TYPE(NO_WIKI_END);
+    SET_TOKEN_TYPE(BLOCKQUOTE);
+    SET_TOKEN_TYPE(BLOCKQUOTE_START);
+    SET_TOKEN_TYPE(BLOCKQUOTE_END);
+    SET_TOKEN_TYPE(STRONG_EM);
+    SET_TOKEN_TYPE(STRONG_START);
+    SET_TOKEN_TYPE(STRONG_END);
+    SET_TOKEN_TYPE(STRONG);
+    SET_TOKEN_TYPE(EM_START);
+    SET_TOKEN_TYPE(EM_END);
+    SET_TOKEN_TYPE(EM);
+    SET_TOKEN_TYPE(TT_START);
+    SET_TOKEN_TYPE(TT_END);
+    SET_TOKEN_TYPE(TT);
+    SET_TOKEN_TYPE(OL);
+    SET_TOKEN_TYPE(UL);
+    SET_TOKEN_TYPE(H1_START);
+    SET_TOKEN_TYPE(H2_START);
+    SET_TOKEN_TYPE(H3_START);
+    SET_TOKEN_TYPE(H4_START);
+    SET_TOKEN_TYPE(H5_START);
+    SET_TOKEN_TYPE(H6_START);
+    SET_TOKEN_TYPE(H1_END);
+    SET_TOKEN_TYPE(H2_END);
+    SET_TOKEN_TYPE(H3_END);
+    SET_TOKEN_TYPE(H4_END);
+    SET_TOKEN_TYPE(H5_END);
+    SET_TOKEN_TYPE(H6_END);
+    SET_TOKEN_TYPE(URI);
+    SET_TOKEN_TYPE(MAIL);
+    SET_TOKEN_TYPE(PATH);
+    SET_TOKEN_TYPE(LINK_START);
+    SET_TOKEN_TYPE(LINK_END);
+    SET_TOKEN_TYPE(EXT_LINK_START);
+    SET_TOKEN_TYPE(EXT_LINK_END);
+    SET_TOKEN_TYPE(SEPARATOR);
+    SET_TOKEN_TYPE(SPACE);
+    SET_TOKEN_TYPE(QUOT_ENTITY);
+    SET_TOKEN_TYPE(AMP_ENTITY);
+    SET_TOKEN_TYPE(NAMED_ENTITY);
+    SET_TOKEN_TYPE(HEX_ENTITY);
+    SET_TOKEN_TYPE(DECIMAL_ENTITY);
+    SET_TOKEN_TYPE(QUOT);
+    SET_TOKEN_TYPE(AMP);
+    SET_TOKEN_TYPE(LESS);
+    SET_TOKEN_TYPE(GREATER);
+    SET_TOKEN_TYPE(IMG_START);
+    SET_TOKEN_TYPE(IMG_END);
+    SET_TOKEN_TYPE(LEFT_CURLY);
+    SET_TOKEN_TYPE(RIGHT_CURLY);
+    SET_TOKEN_TYPE(CRLF);
+    SET_TOKEN_TYPE(SPECIAL_URI_CHARS);
+    SET_TOKEN_TYPE(PRINTABLE);
+    SET_TOKEN_TYPE(ALNUM);
+    SET_TOKEN_TYPE(DEFAULT);
+    SET_TOKEN_TYPE(END_OF_FILE);
+
+#undef SET_TOKEN_TYPE
+
+    return hash;
+}
+
+// for testing and debugging only
+VALUE wiki_token(token_t *token)
+{
+    VALUE object = rb_class_new_instance(0, NULL, cWikitextParserToken);
+    (void)rb_iv_set(object, "@start",           LONG2NUM((long)token->start));
+    (void)rb_iv_set(object, "@stop",            LONG2NUM((long)token->stop));
+    (void)rb_iv_set(object, "@line_start",      LONG2NUM(token->line_start));
+    (void)rb_iv_set(object, "@line_stop",       LONG2NUM(token->line_stop));
+    (void)rb_iv_set(object, "@column_start",    LONG2NUM(token->column_start));
+    (void)rb_iv_set(object, "@column_stop",     LONG2NUM(token->column_stop));
+    (void)rb_iv_set(object, "@code_point",      INT2NUM(token->code_point));
+
+    // look-up the token type
+    VALUE types = Wikitext_parser_token_types(Qnil);
+    VALUE type  = rb_hash_aref(types, INT2FIX(token->type));
+    (void)rb_iv_set(object, "@token_type",      type);
+    (void)rb_iv_set(object, "@string_value",    rb_str_new(token->start, token->stop - token->start));
+    return object;
+}

data/ext/wikitext/token.h

@@ -0,0 +1,117 @@
+// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+#include "ruby_compat.h"
+#include <stdint.h>     /* uint32_t */
+
+#define TOKEN_LEN(token)    (token->stop - token->start)
+#define TOKEN_TEXT(token)   rb_str_new((const char *)token->start, TOKEN_LEN(token))
+
+typedef struct
+{
+    char        *start;
+    char        *stop;
+    size_t      line_start;
+    size_t      line_stop;
+    size_t      column_start;
+    size_t      column_stop;
+    uint32_t    code_point;
+    int         type;
+} token_t;
+
+enum token_types {
+    NO_TOKEN,
+    P,              // imaginary token (never explicitly marked up)
+    LI,             // imaginary token (never explicitly marked up)
+    NESTED_LIST,    // imaginary token (never explicitly marked up)
+    PRE,
+    PRE_START,
+    PRE_END,
+    NO_WIKI_START,
+    NO_WIKI_END,
+    BLOCKQUOTE,
+    BLOCKQUOTE_START,
+    BLOCKQUOTE_END,
+    STRONG_EM,
+    STRONG_START,
+    STRONG_END,
+    STRONG,
+    EM_START,
+    EM_END,
+    EM,
+    TT_START,
+    TT_END,
+    TT,
+    OL,
+    UL,
+
+    // keep these consecutive, and in ascending order
+    // (the arithmetic for the base_heading_level feature assumes this)
+    H1_START,
+    H2_START,
+    H3_START,
+    H4_START,
+    H5_START,
+    H6_START,
+
+    // likewise for the H*_END tokens
+    H1_END,
+    H2_END,
+    H3_END,
+    H4_END,
+    H5_END,
+    H6_END,
+
+    URI,
+    MAIL,
+    PATH,
+    LINK_START,
+    LINK_END,
+    EXT_LINK_START,
+    EXT_LINK_END,
+    SEPARATOR,
+    SPACE,
+    QUOT_ENTITY,
+    AMP_ENTITY,
+    NAMED_ENTITY,
+    HEX_ENTITY,
+    DECIMAL_ENTITY,
+    QUOT,
+    AMP,
+    LESS,
+    GREATER,
+    IMG_START,
+    IMG_END,
+    LEFT_CURLY,
+    RIGHT_CURLY,
+    CRLF,
+    SPECIAL_URI_CHARS,
+    PRINTABLE,
+    ALNUM,
+    DEFAULT,
+    END_OF_FILE
+};
+
+VALUE Wikitext_parser_token_types(VALUE self);
+
+VALUE wiki_token(token_t *token);