wikitext 4.0 → 4.0.1

data/ext/parser.h

@@ -1,40 +0,0 @@
-// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice,
-//    this list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright notice,
-//    this list of conditions and the following disclaimer in the documentation
-//    and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-// POSSIBILITY OF SUCH DAMAGE.
-
-#include "ruby_compat.h"
-
-VALUE Wikitext_parser_initialize(int argc, VALUE *argv, VALUE self);
-
-VALUE Wikitext_parser_tokenize(VALUE self, VALUE string);
-
-VALUE Wikitext_parser_benchmarking_tokenize(VALUE self, VALUE string);
-
-VALUE Wikitext_parser_fulltext_tokenize(int argc, VALUE *argv, VALUE self);
-
-VALUE Wikitext_parser_sanitize_link_target(VALUE self, VALUE string);
-
-VALUE Wikitext_parser_encode_link_target(VALUE self, VALUE in);
-
-VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self);
-
-VALUE Wikitext_parser_profiling_parse(VALUE self, VALUE string);

data/ext/ruby_compat.h

@@ -1,34 +0,0 @@
-// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice,
-//    this list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright notice,
-//    this list of conditions and the following disclaimer in the documentation
-//    and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-// POSSIBILITY OF SUCH DAMAGE.
-
-#include <ruby.h>
-
-// for compatibility with Ruby 1.8.5, which doesn't declare RSTRING_PTR
-#ifndef RSTRING_PTR
-#define RSTRING_PTR(s) (RSTRING(s)->ptr)
-#endif
-
-// for compatibility with Ruby 1.8.5, which doesn't declare RSTRING_LEN
-#ifndef RSTRING_LEN
-#define RSTRING_LEN(s) (RSTRING(s)->len)
-#endif

data/ext/str.c

@@ -1,109 +0,0 @@
-// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice,
-//    this list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright notice,
-//    this list of conditions and the following disclaimer in the documentation
-//    and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-// POSSIBILITY OF SUCH DAMAGE.
-
-#include "str.h"
-
-// when allocating memory, reserve a little more than was asked for,
-// which can help to avoid subsequent allocations
-#define STR_OVERALLOC 256
-
-str_t *str_new(void)
-{
-    str_t *str      = ALLOC_N(str_t, 1);
-    str->ptr        = NULL;
-    str->len        = 0;
-    str->capacity   = 0;
-    return str;
-}
-
-str_t *str_new_copy(const char *src, long len)
-{
-    str_t *str      = ALLOC_N(str_t, 1);
-    str->ptr        = ALLOC_N(char, len + STR_OVERALLOC);
-    memcpy(str->ptr, src, len);
-    str->len        = len;
-    str->capacity   = len + STR_OVERALLOC;
-    return str;
-}
-
-str_t *str_new_from_string(VALUE string)
-{
-    string = StringValue(string);
-    return str_new_copy(RSTRING_PTR(string), RSTRING_LEN(string));
-}
-
-VALUE string_from_str(str_t *str)
-{
-    VALUE string = rb_str_new(str->ptr, str->len);
-    rb_funcall(string, rb_intern("force_encoding"), 1, rb_str_new2("UTF-8"));
-    return string;
-}
-
-void str_grow(str_t *str, long len)
-{
-    if (str->capacity < len)
-    {
-        if (str->ptr)
-            REALLOC_N(str->ptr, char, len + STR_OVERALLOC);
-        else
-            str->ptr = ALLOC_N(char, len + STR_OVERALLOC);
-        str->capacity = len + STR_OVERALLOC;
-    }
-}
-
-void str_append(str_t *str, const char *src, long len)
-{
-    long new_len = str->len + len;
-    if (str->capacity < new_len)
-    {
-        if (str->ptr)
-            REALLOC_N(str->ptr, char, new_len + STR_OVERALLOC);
-        else
-            str->ptr = ALLOC_N(char, new_len + STR_OVERALLOC);
-        str->capacity = new_len + STR_OVERALLOC;
-    }
-    memcpy(str->ptr + str->len, src, len);
-    str->len = new_len;
-}
-
-void str_append_str(str_t *str, str_t *other)
-{
-    str_append(str, other->ptr, other->len);
-}
-
-void str_append_string(str_t *str, VALUE other)
-{
-    str_append(str, RSTRING_PTR(other), RSTRING_LEN(other));
-}
-
-void str_clear(str_t *str)
-{
-    str->len = 0;
-}
-
-void str_free(str_t *str)
-{
-    if (str->ptr)
-        free(str->ptr);
-    free(str);
-}

data/ext/str.h

@@ -1,64 +0,0 @@
-// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice,
-//    this list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright notice,
-//    this list of conditions and the following disclaimer in the documentation
-//    and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-// POSSIBILITY OF SUCH DAMAGE.
-
-#include "ruby_compat.h"
-
-typedef struct
-{
-    char *ptr;
-    long len;
-    long capacity;
-} str_t;
-
-// Mark the str struct designated by ptr as a participant in Ruby's mark-and-sweep garbage collection scheme.
-// A variable named name is placed on the C stack to prevent the structure from being prematurely collected.
-#define GC_WRAP_STR(ptr, name) volatile VALUE name __attribute__((unused)) = Data_Wrap_Struct(rb_cObject, 0, str_free, ptr)
-
-// create a new, empty string struct
-str_t *str_new(void);
-
-// create a new string struct and initialize it with a copy of the buffer of length len pointed to by src
-str_t *str_new_copy(const char *src, long len);
-
-// convenience method for testing
-str_t *str_new_from_string(VALUE string);
-
-// convenience method for testing
-VALUE string_from_str(str_t *str);
-
-// grows a string's capacity to the specified length
-void str_grow(str_t *str, long len);
-
-void str_append(str_t *str, const char *src, long len);
-
-// appends the "other" string struct onto str
-void str_append_str(str_t *str, str_t *other);
-
-// appends the "other" string (a Ruby String) onto str
-void str_append_string(str_t *str, VALUE other);
-
-// don't actually free the memory yet
-// this makes str structs very useful when reusing buffers because it avoids reallocation
-void str_clear(str_t *str);
-
-void str_free(str_t *str);

data/ext/token.c

@@ -1,125 +0,0 @@
-// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice,
-//    this list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright notice,
-//    this list of conditions and the following disclaimer in the documentation
-//    and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-// POSSIBILITY OF SUCH DAMAGE.
-
-#include "token.h"
-#include "wikitext.h"
-
-// return a hash of token types
-// we make this available for unit testing purposes
-
-VALUE Wikitext_parser_token_types(VALUE self)
-{
-    VALUE hash = rb_hash_new();
-
-#define SET_TOKEN_TYPE(identifier)  (void)rb_hash_aset(hash, INT2FIX(identifier), \
-    rb_funcall(rb_funcall(rb_str_new2(#identifier), rb_intern("downcase"), 0), rb_intern("to_sym"), 0))
-
-    SET_TOKEN_TYPE(NO_TOKEN);
-    SET_TOKEN_TYPE(P);
-    SET_TOKEN_TYPE(LI);
-    SET_TOKEN_TYPE(NESTED_LIST);
-    SET_TOKEN_TYPE(PRE);
-    SET_TOKEN_TYPE(PRE_START);
-    SET_TOKEN_TYPE(PRE_END);
-    SET_TOKEN_TYPE(NO_WIKI_START);
-    SET_TOKEN_TYPE(NO_WIKI_END);
-    SET_TOKEN_TYPE(BLOCKQUOTE);
-    SET_TOKEN_TYPE(BLOCKQUOTE_START);
-    SET_TOKEN_TYPE(BLOCKQUOTE_END);
-    SET_TOKEN_TYPE(STRONG_EM);
-    SET_TOKEN_TYPE(STRONG_START);
-    SET_TOKEN_TYPE(STRONG_END);
-    SET_TOKEN_TYPE(STRONG);
-    SET_TOKEN_TYPE(EM_START);
-    SET_TOKEN_TYPE(EM_END);
-    SET_TOKEN_TYPE(EM);
-    SET_TOKEN_TYPE(TT_START);
-    SET_TOKEN_TYPE(TT_END);
-    SET_TOKEN_TYPE(TT);
-    SET_TOKEN_TYPE(OL);
-    SET_TOKEN_TYPE(UL);
-    SET_TOKEN_TYPE(H1_START);
-    SET_TOKEN_TYPE(H2_START);
-    SET_TOKEN_TYPE(H3_START);
-    SET_TOKEN_TYPE(H4_START);
-    SET_TOKEN_TYPE(H5_START);
-    SET_TOKEN_TYPE(H6_START);
-    SET_TOKEN_TYPE(H1_END);
-    SET_TOKEN_TYPE(H2_END);
-    SET_TOKEN_TYPE(H3_END);
-    SET_TOKEN_TYPE(H4_END);
-    SET_TOKEN_TYPE(H5_END);
-    SET_TOKEN_TYPE(H6_END);
-    SET_TOKEN_TYPE(URI);
-    SET_TOKEN_TYPE(MAIL);
-    SET_TOKEN_TYPE(PATH);
-    SET_TOKEN_TYPE(LINK_START);
-    SET_TOKEN_TYPE(LINK_END);
-    SET_TOKEN_TYPE(EXT_LINK_START);
-    SET_TOKEN_TYPE(EXT_LINK_END);
-    SET_TOKEN_TYPE(SEPARATOR);
-    SET_TOKEN_TYPE(SPACE);
-    SET_TOKEN_TYPE(QUOT_ENTITY);
-    SET_TOKEN_TYPE(AMP_ENTITY);
-    SET_TOKEN_TYPE(NAMED_ENTITY);
-    SET_TOKEN_TYPE(HEX_ENTITY);
-    SET_TOKEN_TYPE(DECIMAL_ENTITY);
-    SET_TOKEN_TYPE(QUOT);
-    SET_TOKEN_TYPE(AMP);
-    SET_TOKEN_TYPE(LESS);
-    SET_TOKEN_TYPE(GREATER);
-    SET_TOKEN_TYPE(IMG_START);
-    SET_TOKEN_TYPE(IMG_END);
-    SET_TOKEN_TYPE(LEFT_CURLY);
-    SET_TOKEN_TYPE(RIGHT_CURLY);
-    SET_TOKEN_TYPE(CRLF);
-    SET_TOKEN_TYPE(SPECIAL_URI_CHARS);
-    SET_TOKEN_TYPE(PRINTABLE);
-    SET_TOKEN_TYPE(ALNUM);
-    SET_TOKEN_TYPE(DEFAULT);
-    SET_TOKEN_TYPE(END_OF_FILE);
-
-#undef SET_TOKEN_TYPE
-
-    return hash;
-}
-
-// for testing and debugging only
-VALUE wiki_token(token_t *token)
-{
-    VALUE object = rb_class_new_instance(0, NULL, cWikitextParserToken);
-    (void)rb_iv_set(object, "@start",           LONG2NUM((long)token->start));
-    (void)rb_iv_set(object, "@stop",            LONG2NUM((long)token->stop));
-    (void)rb_iv_set(object, "@line_start",      LONG2NUM(token->line_start));
-    (void)rb_iv_set(object, "@line_stop",       LONG2NUM(token->line_stop));
-    (void)rb_iv_set(object, "@column_start",    LONG2NUM(token->column_start));
-    (void)rb_iv_set(object, "@column_stop",     LONG2NUM(token->column_stop));
-    (void)rb_iv_set(object, "@code_point",      INT2NUM(token->code_point));
-
-    // look-up the token type
-    VALUE types = Wikitext_parser_token_types(Qnil);
-    VALUE type  = rb_hash_aref(types, INT2FIX(token->type));
-    (void)rb_iv_set(object, "@token_type",      type);
-    (void)rb_iv_set(object, "@string_value",    rb_str_new(token->start, token->stop - token->start));
-    return object;
-}

data/ext/token.h

@@ -1,117 +0,0 @@
-// Copyright 2008-2009 Wincent Colaiuta. All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice,
-//    this list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright notice,
-//    this list of conditions and the following disclaimer in the documentation
-//    and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
-// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-// POSSIBILITY OF SUCH DAMAGE.
-
-#include "ruby_compat.h"
-#include <stdint.h>     /* uint32_t */
-
-#define TOKEN_LEN(token)    (token->stop - token->start)
-#define TOKEN_TEXT(token)   rb_str_new((const char *)token->start, TOKEN_LEN(token))
-
-typedef struct
-{
-    char        *start;
-    char        *stop;
-    size_t      line_start;
-    size_t      line_stop;
-    size_t      column_start;
-    size_t      column_stop;
-    uint32_t    code_point;
-    int         type;
-} token_t;
-
-enum token_types {
-    NO_TOKEN,
-    P,              // imaginary token (never explicitly marked up)
-    LI,             // imaginary token (never explicitly marked up)
-    NESTED_LIST,    // imaginary token (never explicitly marked up)
-    PRE,
-    PRE_START,
-    PRE_END,
-    NO_WIKI_START,
-    NO_WIKI_END,
-    BLOCKQUOTE,
-    BLOCKQUOTE_START,
-    BLOCKQUOTE_END,
-    STRONG_EM,
-    STRONG_START,
-    STRONG_END,
-    STRONG,
-    EM_START,
-    EM_END,
-    EM,
-    TT_START,
-    TT_END,
-    TT,
-    OL,
-    UL,
-
-    // keep these consecutive, and in ascending order
-    // (the arithmetic for the base_heading_level feature assumes this)
-    H1_START,
-    H2_START,
-    H3_START,
-    H4_START,
-    H5_START,
-    H6_START,
-
-    // likewise for the H*_END tokens
-    H1_END,
-    H2_END,
-    H3_END,
-    H4_END,
-    H5_END,
-    H6_END,
-
-    URI,
-    MAIL,
-    PATH,
-    LINK_START,
-    LINK_END,
-    EXT_LINK_START,
-    EXT_LINK_END,
-    SEPARATOR,
-    SPACE,
-    QUOT_ENTITY,
-    AMP_ENTITY,
-    NAMED_ENTITY,
-    HEX_ENTITY,
-    DECIMAL_ENTITY,
-    QUOT,
-    AMP,
-    LESS,
-    GREATER,
-    IMG_START,
-    IMG_END,
-    LEFT_CURLY,
-    RIGHT_CURLY,
-    CRLF,
-    SPECIAL_URI_CHARS,
-    PRINTABLE,
-    ALNUM,
-    DEFAULT,
-    END_OF_FILE
-};
-
-VALUE Wikitext_parser_token_types(VALUE self);
-
-VALUE wiki_token(token_t *token);