wikitext 0.1

data/ext/parser.h

@@ -0,0 +1,31 @@
+// Copyright 2008 Wincent Colaiuta
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include <ruby/ruby.h>
+
+VALUE Wikitext_parser_initialize(VALUE self);
+
+VALUE Wikitext_parser_tokenize(VALUE self, VALUE string);
+
+VALUE Wikitext_parser_benchmarking_tokenize(VALUE self, VALUE string);
+
+VALUE Wikitext_parser_sanitize_link_target(VALUE self, VALUE string);
+
+VALUE Wikitext_parser_encode_link_target(VALUE self, VALUE in);
+
+VALUE Wikitext_parser_encode_special_link_target(VALUE self, VALUE in);
+
+VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self);
+
+VALUE Wikitext_parser_profiling_parse(VALUE self, VALUE string);

data/ext/str.h

@@ -0,0 +1,135 @@
+// Copyright 2008 Wincent Colaiuta
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include <ruby/ruby.h>
+
+typedef struct
+{
+    char *ptr;
+    long len;
+    long capacity;
+} str_t;
+
+// create a new, empty string struct
+inline str_t *str_new(void)
+{
+    str_t *str      = ALLOC_N(str_t, 1);
+    str->ptr        = NULL;
+    str->len        = 0;
+    str->capacity   = 0;
+    return str;
+}
+
+// create a new, empty string struct with capacity len
+inline str_t *str_new_size(long len)
+{
+    str_t *str      = ALLOC_N(str_t, 1);
+    str->ptr        = ALLOC_N(char, len);
+    str->len        = 0;
+    str->capacity   = len;
+    return str;
+}
+
+// create a new string struct and initialize it with a copy of the buffer of length len pointed to by src
+inline str_t *str_new_copy(char *src, long len)
+{
+    str_t *str      = ALLOC_N(str_t, 1);
+    str->ptr        = ALLOC_N(char, len);
+    memcpy(str->ptr, src, len);
+    str->len        = len;
+    str->capacity   = len;
+    return str;
+}
+
+// create a new string struct and initialize it with the buffer of length len pointed to by src
+// no copy is made; the struct takes ownership of the buffer and will free it when the struct is disposed of
+inline str_t *str_new_no_copy(char *src, long len)
+{
+    str_t *str      = ALLOC_N(str_t, 1);
+    str->ptr        = src;
+    str->len        = len;
+    str->capacity   = len;
+    return str;
+}
+
+// convenience method for testing
+inline str_t *str_new_from_string(VALUE string)
+{
+    string = StringValue(string);
+    return str_new_copy(RSTRING_PTR(string), RSTRING_LEN(string));
+}
+
+// convenience method for testing
+inline VALUE string_from_str(str_t *str)
+{
+    return rb_str_new(str->ptr, str->len);
+}
+
+// grows a string's capacity to the specified length
+inline void str_grow(str_t *str, long len)
+{
+    if (str->capacity < len)
+    {
+        if (str->ptr)
+            REALLOC_N(str->ptr, char, len);
+        else
+            str->ptr = ALLOC_N(char, len);
+        str->capacity = len;
+    }
+}
+
+inline void str_append(str_t *str, char *src, long len)
+{
+    long new_len = str->len + len;
+    if (str->capacity < new_len)
+    {
+        if (str->ptr)
+            REALLOC_N(str->ptr, char, new_len);
+        else
+            str->ptr = ALLOC_N(char, new_len);
+        str->capacity = new_len;
+    }
+    memcpy(str->ptr + str->len, src, len);
+    str->len = new_len;
+}
+
+// appends the "other" string struct onto str
+inline void str_append_str(str_t *str, str_t *other)
+{
+    str_append(str, other->ptr, other->len);
+}
+
+// this is a temporary convenience measure
+// later on if I develop in-place variants of some functions this won't be needed
+inline void str_swap(str_t **a, str_t **b)
+{
+    str_t *c;
+    c = *a;
+    *a = *b;
+    *b = c;
+}
+
+// don't actually free the memory yet
+// this makes str structs very useful when reusing buffers because it avoids reallocation
+inline void str_clear(str_t *str)
+{
+    str->len = 0;
+}
+
+inline void str_free(str_t *str)
+{
+    if (str->ptr)
+        free(str->ptr);
+    free(str);
+}

data/ext/token.c

@@ -0,0 +1,109 @@
+// Copyright 2008 Wincent Colaiuta
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "token.h"
+#include "wikitext.h"
+
+// return a hash of token types
+// we make this available for unit testing purposes
+
+VALUE Wikitext_parser_token_types(VALUE self)
+{
+    VALUE hash = rb_hash_new();
+
+#define SET_TOKEN_TYPE(identifier)  (void)rb_hash_aset(hash, INT2FIX(identifier), \
+    rb_funcall(rb_funcall(rb_str_new2(#identifier), rb_intern("downcase"), 0), rb_intern("to_sym"), 0))
+
+    SET_TOKEN_TYPE(NO_TOKEN);
+    SET_TOKEN_TYPE(P);
+    SET_TOKEN_TYPE(LI);
+    SET_TOKEN_TYPE(NESTED_LIST);
+    SET_TOKEN_TYPE(PRE);
+    SET_TOKEN_TYPE(PRE_START);
+    SET_TOKEN_TYPE(PRE_END);
+    SET_TOKEN_TYPE(NO_WIKI_START);
+    SET_TOKEN_TYPE(NO_WIKI_END);
+    SET_TOKEN_TYPE(BLOCKQUOTE);
+    SET_TOKEN_TYPE(BLOCKQUOTE_START);
+    SET_TOKEN_TYPE(BLOCKQUOTE_END);
+    SET_TOKEN_TYPE(STRONG_EM);
+    SET_TOKEN_TYPE(STRONG_START);
+    SET_TOKEN_TYPE(STRONG_END);
+    SET_TOKEN_TYPE(STRONG);
+    SET_TOKEN_TYPE(EM_START);
+    SET_TOKEN_TYPE(EM_END);
+    SET_TOKEN_TYPE(EM);
+    SET_TOKEN_TYPE(TT_START);
+    SET_TOKEN_TYPE(TT_END);
+    SET_TOKEN_TYPE(TT);
+    SET_TOKEN_TYPE(OL);
+    SET_TOKEN_TYPE(UL);
+    SET_TOKEN_TYPE(H6_START);
+    SET_TOKEN_TYPE(H5_START);
+    SET_TOKEN_TYPE(H4_START);
+    SET_TOKEN_TYPE(H3_START);
+    SET_TOKEN_TYPE(H2_START);
+    SET_TOKEN_TYPE(H1_START);
+    SET_TOKEN_TYPE(H6_END);
+    SET_TOKEN_TYPE(H5_END);
+    SET_TOKEN_TYPE(H4_END);
+    SET_TOKEN_TYPE(H3_END);
+    SET_TOKEN_TYPE(H2_END);
+    SET_TOKEN_TYPE(H1_END);
+    SET_TOKEN_TYPE(URI);
+    SET_TOKEN_TYPE(MAIL);
+    SET_TOKEN_TYPE(LINK_START);
+    SET_TOKEN_TYPE(LINK_END);
+    SET_TOKEN_TYPE(EXT_LINK_START);
+    SET_TOKEN_TYPE(EXT_LINK_END);
+    SET_TOKEN_TYPE(SEPARATOR);
+    SET_TOKEN_TYPE(SPACE);
+    SET_TOKEN_TYPE(QUOT_ENTITY);
+    SET_TOKEN_TYPE(AMP_ENTITY);
+    SET_TOKEN_TYPE(NAMED_ENTITY);
+    SET_TOKEN_TYPE(HEX_ENTITY);
+    SET_TOKEN_TYPE(DECIMAL_ENTITY);
+    SET_TOKEN_TYPE(QUOT);
+    SET_TOKEN_TYPE(AMP);
+    SET_TOKEN_TYPE(LESS);
+    SET_TOKEN_TYPE(GREATER);
+    SET_TOKEN_TYPE(CRLF);
+    SET_TOKEN_TYPE(PRINTABLE);
+    SET_TOKEN_TYPE(DEFAULT);
+    SET_TOKEN_TYPE(END_OF_FILE);
+
+#undef SET_TOKEN_TYPE
+
+    return hash;
+}
+
+// for testing and debugging only
+VALUE _Wikitext_token(token_t *token)
+{
+    VALUE object = rb_class_new_instance(0, NULL, cWikitextParserToken);
+    (void)rb_iv_set(object, "@start",           LONG2NUM((long)token->start));
+    (void)rb_iv_set(object, "@stop",            LONG2NUM((long)token->stop));
+    (void)rb_iv_set(object, "@line_start",      LONG2NUM(token->line_start));
+    (void)rb_iv_set(object, "@line_stop",       LONG2NUM(token->line_stop));
+    (void)rb_iv_set(object, "@column_start",    LONG2NUM(token->column_start));
+    (void)rb_iv_set(object, "@column_stop",     LONG2NUM(token->column_stop));
+    (void)rb_iv_set(object, "@code_point",      INT2NUM(token->code_point));
+
+    // look-up the token type
+    VALUE types = Wikitext_parser_token_types(Qnil);
+    VALUE type  = rb_hash_aref(types, INT2FIX(token->type));
+    (void)rb_iv_set(object, "@token_type",      type);
+    (void)rb_iv_set(object, "@string_value",    rb_str_new(token->start, token->stop - token->start));
+    return object;
+}

data/ext/token.h

@@ -0,0 +1,95 @@
+// Copyright 2008 Wincent Colaiuta
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include <ruby/ruby.h>
+#include <stdint.h>     /* uint32_t */
+
+#define TOKEN_TEXT(token)   rb_str_new((const char *)token->start, (token->stop - token->start))
+#define TOKEN_LEN(token)    (token->stop - token->start)
+
+typedef struct
+{
+    char        *start;
+    char        *stop;
+    size_t      line_start;
+    size_t      line_stop;
+    size_t      column_start;
+    size_t      column_stop;
+    uint32_t    code_point;
+    int         type;
+} token_t;
+
+enum token_types {
+    NO_TOKEN,
+    P,              // imaginary token (never explicitly marked up)
+    LI,             // imaginary token (never explicitly marked up)
+    NESTED_LIST,    // imaginary token (never explicitly marked up)
+    PRE,
+    PRE_START,
+    PRE_END,
+    NO_WIKI_START,
+    NO_WIKI_END,
+    BLOCKQUOTE,
+    BLOCKQUOTE_START,
+    BLOCKQUOTE_END,
+    STRONG_EM,
+    STRONG_START,
+    STRONG_END,
+    STRONG,
+    EM_START,
+    EM_END,
+    EM,
+    TT_START,
+    TT_END,
+    TT,
+    OL,
+    UL,
+    H6_START,
+    H5_START,
+    H4_START,
+    H3_START,
+    H2_START,
+    H1_START,
+    H6_END,
+    H5_END,
+    H4_END,
+    H3_END,
+    H2_END,
+    H1_END,
+    URI,
+    MAIL,
+    LINK_START,
+    LINK_END,
+    EXT_LINK_START,
+    EXT_LINK_END,
+    SEPARATOR,
+    SPACE,
+    QUOT_ENTITY,
+    AMP_ENTITY,
+    NAMED_ENTITY,
+    HEX_ENTITY,
+    DECIMAL_ENTITY,
+    QUOT,
+    AMP,
+    LESS,
+    GREATER,
+    CRLF,
+    PRINTABLE,
+    DEFAULT,
+    END_OF_FILE
+};
+
+VALUE Wikitext_parser_token_types(VALUE self);
+
+VALUE _Wikitext_token(token_t *token);

data/ext/wikitext.c

@@ -0,0 +1,60 @@
+// Copyright 2008 Wincent Colaiuta
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "wikitext_ragel.h"
+#include "parser.h"
+
+VALUE mWikitext              = 0;   // module Wikitext
+VALUE cWikitextParser        = 0;   // class Wikitext::Parser
+VALUE eWikitextParserError   = 0;   // class Wikitext::Parser::Error
+VALUE cWikitextParserToken   = 0;   // class Wikitext::Parser::Token
+
+void Init_wikitext()
+{
+    // Wikitext
+    mWikitext = rb_define_module("Wikitext");
+
+    // Wikitext::Parser
+    cWikitextParser = rb_define_class_under(mWikitext, "Parser", rb_cObject);
+    rb_define_method(cWikitextParser, "initialize", Wikitext_parser_initialize, 0);
+    rb_define_method(cWikitextParser, "parse", Wikitext_parser_parse, -1);
+    rb_define_method(cWikitextParser, "profiling_parse", Wikitext_parser_profiling_parse, 1);
+    rb_define_method(cWikitextParser, "tokenize", Wikitext_parser_tokenize, 1);
+    rb_define_method(cWikitextParser, "benchmarking_tokenize", Wikitext_parser_benchmarking_tokenize, 1);
+    rb_define_singleton_method(cWikitextParser, "sanitize_link_target", Wikitext_parser_sanitize_link_target, 1);
+    rb_define_singleton_method(cWikitextParser, "encode_link_target", Wikitext_parser_encode_link_target, 1);
+    rb_define_singleton_method(cWikitextParser, "encode_special_link_target", Wikitext_parser_encode_special_link_target, 1);
+    rb_define_attr(cWikitextParser, "line_ending", Qtrue, Qtrue);
+    rb_define_attr(cWikitextParser, "internal_link_prefix", Qtrue, Qtrue);
+    rb_define_attr(cWikitextParser, "external_link_class", Qtrue, Qtrue);
+    rb_define_attr(cWikitextParser, "mailto_class", Qtrue, Qtrue);
+    rb_define_attr(cWikitextParser, "autolink", Qtrue, Qtrue);
+    rb_define_attr(cWikitextParser, "treat_slash_as_special", Qtrue, Qtrue);
+
+    // Wikitext::Parser::Error
+    eWikitextParserError = rb_define_class_under(cWikitextParser, "Error", rb_eException);
+
+    // Wikitext::Parser::Token
+    cWikitextParserToken = rb_define_class_under(cWikitextParser, "Token", rb_cObject);
+    rb_define_singleton_method(cWikitextParserToken, "types", Wikitext_parser_token_types, 0);
+    rb_define_attr(cWikitextParserToken, "start", Qtrue, Qfalse);
+    rb_define_attr(cWikitextParserToken, "stop", Qtrue, Qfalse);
+    rb_define_attr(cWikitextParserToken, "line_start", Qtrue, Qfalse);
+    rb_define_attr(cWikitextParserToken, "line_stop", Qtrue, Qfalse);
+    rb_define_attr(cWikitextParserToken, "column_start", Qtrue, Qfalse);
+    rb_define_attr(cWikitextParserToken, "column_stop", Qtrue, Qfalse);
+    rb_define_attr(cWikitextParserToken, "code_point", Qtrue, Qfalse);
+    rb_define_attr(cWikitextParserToken, "token_type", Qtrue, Qfalse);
+    rb_define_attr(cWikitextParserToken, "string_value", Qtrue, Qfalse);
+}

data/ext/wikitext.h

@@ -0,0 +1,30 @@
+// Copyright 2008 Wincent Colaiuta
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include <ruby/ruby.h>
+#include <stdint.h>
+
+#define ruby_inspect(obj) rb_funcall(rb_mKernel, rb_intern("p"), 1, obj)
+
+// module Wikitext
+extern VALUE mWikitext;
+
+// class Wikitext::Parser
+extern VALUE cWikitextParser;
+
+// class Wikitext::Parser::Error
+extern VALUE eWikitextParserError;
+
+// class Wikitext::Parser::Token
+extern VALUE cWikitextParserToken;