wikitext 1.0 → 1.0.1

data/ext/parser.c

@@ -23,6 +23,7 @@
 // poor man's object orientation in C:
 // instead of parsing around multiple parameters between functions in the parser
 // we pack everything into a struct and pass around only a pointer to that
+// TODO: consider changing some of the VALUE members (eg link_target) to the more efficient str_t type
 typedef struct
 {
     VALUE   output;                 // for accumulating output to be returned
@@ -420,6 +421,15 @@ void _Wikitext_start_para_if_necessary(parser_t *parser)
     parser->pending_crlf = Qfalse;
 }
 
+void _Wikitext_emit_pending_crlf_if_necessary(parser_t *parser)
+{
+    if (parser->pending_crlf == Qtrue)
+    {
+        rb_str_cat(parser->output, parser->line_ending->ptr, parser->line_ending->len);
+        parser->pending_crlf = Qfalse;
+    }
+}
+
 // Helper function that pops any excess elements off scope (pushing is already handled in the respective rules).
 // For example, given input like:
 //
@@ -1035,7 +1045,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case PRE_START:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_pre_start, sizeof(escaped_pre_start) - 1);
+                }
                 else if (IN(BLOCKQUOTE_START))
                 {
                     _Wikitext_rollback_failed_link(parser);             // if any
@@ -1046,9 +1059,16 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
                     ary_push(parser->scope, PRE_START);
                     ary_push(parser->line, PRE_START);
                 }
-                else if (parser->scope->count == 0 || (IN(P) && !IN(BLOCKQUOTE)))
+                else if (IN(BLOCKQUOTE))
+                {
+                    // PRE_START is illegal
+                    i = NIL_P(parser->capture) ? parser->output : parser->capture;
+                    _Wikitext_pop_excess_elements(parser);
+                    _Wikitext_start_para_if_necessary(parser);
+                    rb_str_cat(i, escaped_pre_start, sizeof(escaped_pre_start) - 1);
+                }
+                else
                 {
-                    // would be nice to eliminate the repetition here but it's probably the clearest way
                     _Wikitext_rollback_failed_link(parser);             // if any
                     _Wikitext_rollback_failed_external_link(parser);    // if any
                     _Wikitext_pop_from_stack_up_to(parser, Qnil, P, Qtrue);
@@ -1057,19 +1077,14 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
                     ary_push(parser->scope, PRE_START);
                     ary_push(parser->line, PRE_START);
                 }
-                else
-                {
-                    // everywhere else, PRE_START is illegal (in LI, BLOCKQUOTE, H1_START etc)
-                    i = NIL_P(parser->capture) ? parser->output : parser->capture;
-                    _Wikitext_pop_excess_elements(parser);
-                    _Wikitext_start_para_if_necessary(parser);
-                    rb_str_cat(i, escaped_pre_start, sizeof(escaped_pre_start) - 1);
-                }
                 break;
 
             case PRE_END:
                 if (IN(NO_WIKI_START) || IN(PRE))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_pre_end, sizeof(escaped_pre_end) - 1);
+                }
                 else
                 {
                     if (IN(PRE_START))
@@ -1139,7 +1154,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case BLOCKQUOTE_START:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_blockquote_start, sizeof(escaped_blockquote_start) - 1);
+                }
                 else if (IN(BLOCKQUOTE_START))
                 {
                     // nesting is fine here
@@ -1176,7 +1194,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case BLOCKQUOTE_END:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_blockquote_end, sizeof(escaped_blockquote_end) - 1);
+                }
                 else
                 {
                     if (IN(BLOCKQUOTE_START))
@@ -1193,7 +1214,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case NO_WIKI_START:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_no_wiki_start, sizeof(escaped_no_wiki_start) - 1);
+                }
                 else
                 {
                     _Wikitext_pop_excess_elements(parser);
@@ -1218,6 +1242,7 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
             case STRONG_EM:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
                 {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, literal_strong_em, sizeof(literal_strong_em) - 1);
                     break;
                 }
@@ -1282,7 +1307,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case STRONG:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, literal_strong, sizeof(literal_strong) - 1);
+                }
                 else
                 {
                     i = NIL_P(parser->capture) ? parser->output : parser->capture;
@@ -1306,7 +1334,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case STRONG_START:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_strong_start, sizeof(escaped_strong_start) - 1);
+                }
                 else
                 {
                     i = NIL_P(parser->capture) ? parser->output : parser->capture;
@@ -1325,7 +1356,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case STRONG_END:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_strong_end, sizeof(escaped_strong_end) - 1);
+                }
                 else
                 {
                     i = NIL_P(parser->capture) ? parser->output : parser->capture;
@@ -1343,7 +1377,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case EM:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, literal_em, sizeof(literal_em) - 1);
+                }
                 else
                 {
                     i = NIL_P(parser->capture) ? parser->output : parser->capture;
@@ -1367,7 +1404,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case EM_START:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_em_start, sizeof(escaped_em_start) - 1);
+                }
                 else
                 {
                     i = NIL_P(parser->capture) ? parser->output : parser->capture;
@@ -1386,7 +1426,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case EM_END:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_em_end, sizeof(escaped_em_end) - 1);
+                }
                 else
                 {
                     i = NIL_P(parser->capture) ? parser->output : parser->capture;
@@ -1404,7 +1447,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case TT:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, backtick, sizeof(backtick) - 1);
+                }
                 else
                 {
                     i = NIL_P(parser->capture) ? parser->output : parser->capture;
@@ -1428,7 +1474,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case TT_START:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_tt_start, sizeof(escaped_tt_start) - 1);
+                }
                 else
                 {
                     i = NIL_P(parser->capture) ? parser->output : parser->capture;
@@ -1447,7 +1496,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case TT_END:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, escaped_tt_end, sizeof(escaped_tt_end) - 1);
+                }
                 else
                 {
                     i = NIL_P(parser->capture) ? parser->output : parser->capture;
@@ -1650,7 +1702,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case H6_END:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, literal_h6, sizeof(literal_h6) - 1);
+                }
                 else
                 {
                     _Wikitext_rollback_failed_external_link(parser); // if any
@@ -1665,7 +1720,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case H5_END:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, literal_h5, sizeof(literal_h5) - 1);
+                }
                 else
                 {
                     _Wikitext_rollback_failed_external_link(parser); // if any
@@ -1680,7 +1738,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case H4_END:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, literal_h4, sizeof(literal_h4) - 1);
+                }
                 else
                 {
                     _Wikitext_rollback_failed_external_link(parser); // if any
@@ -1695,7 +1756,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case H3_END:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, literal_h3, sizeof(literal_h3) - 1);
+                }
                 else
                 {
                     _Wikitext_rollback_failed_external_link(parser); // if any
@@ -1710,7 +1774,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case H2_END:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, literal_h2, sizeof(literal_h2) - 1);
+                }
                 else
                 {
                     _Wikitext_rollback_failed_external_link(parser); // if any
@@ -1725,7 +1792,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case H1_END:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, literal_h1, sizeof(literal_h1) - 1);
+                }
                 else
                 {
                     _Wikitext_rollback_failed_external_link(parser); // if any
@@ -1740,7 +1810,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case MAIL:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, token->start, TOKEN_LEN(token));
+                }
                 else
                 {
                     // in plain scope, will turn into autolink (with appropriate, user-configurable CSS)
@@ -1839,12 +1912,15 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
             //          example [[foo €]]
             //          renders <a href="/wiki/Foo_%E2%82%AC">foo €</a>
             // we'll impose similar restrictions here for the link target; allowed tokens will be:
-            //      SPACE, PRINTABLE, DEFAULT, QUOT and AMP
+            //      SPACE, SPECIAL_URI_CHARS, PRINTABLE, DEFAULT, QUOT and AMP
             // everything else will be rejected
             case LINK_START:
                 i = NIL_P(parser->capture) ? parser->output : parser->capture;
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(i, link_start, sizeof(link_start) - 1);
+                }
                 else if (IN(EXT_LINK_START))
                     // already in external link scope! (and in fact, must be capturing link_text right now)
                     rb_str_cat(i, link_start, sizeof(link_start) - 1);
@@ -1868,16 +1944,17 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
                     // look ahead and try to gobble up link target
                     while (NEXT_TOKEN(), (type = token->type))
                     {
-                        if (type == SPACE       ||
-                            type == PRINTABLE   ||
-                            type == DEFAULT     ||
-                            type == QUOT        ||
-                            type == QUOT_ENTITY ||
-                            type == AMP         ||
-                            type == AMP_ENTITY  ||
-                            type == IMG_START   ||
-                            type == IMG_END     ||
-                            type == LEFT_CURLY  ||
+                        if (type == SPACE               ||
+                            type == SPECIAL_URI_CHARS   ||
+                            type == PRINTABLE           ||
+                            type == DEFAULT             ||
+                            type == QUOT                ||
+                            type == QUOT_ENTITY         ||
+                            type == AMP                 ||
+                            type == AMP_ENTITY          ||
+                            type == IMG_START           ||
+                            type == IMG_END             ||
+                            type == LEFT_CURLY          ||
                             type == RIGHT_CURLY)
                         {
                             // accumulate these tokens into link_target
@@ -1920,7 +1997,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
             case LINK_END:
                 i = NIL_P(parser->capture) ? parser->output : parser->capture;
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(i, link_end, sizeof(link_end) - 1);
+                }
                 else if (IN(EXT_LINK_START))
                     // already in external link scope! (and in fact, must be capturing link_text right now)
                     rb_str_cat(i, link_end, sizeof(link_end) - 1);
@@ -1958,7 +2038,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
             case EXT_LINK_START:
                 i = NIL_P(parser->capture) ? parser->output : parser->capture;
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(i, ext_link_start, sizeof(ext_link_start) - 1);
+                }
                 else if (IN(EXT_LINK_START))
                     // already in external link scope! (and in fact, must be capturing link_text right now)
                     rb_str_cat(i, ext_link_start, sizeof(ext_link_start) - 1);
@@ -2003,7 +2086,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
             case EXT_LINK_END:
                 i = NIL_P(parser->capture) ? parser->output : parser->capture;
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(i, ext_link_end, sizeof(ext_link_end) - 1);
+                }
                 else if (IN(EXT_LINK_START))
                 {
                     if (NIL_P(parser->link_text))
@@ -2038,7 +2124,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
             case SPACE:
                 i = NIL_P(parser->capture) ? parser->output : parser->capture;
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(i, token->start, TOKEN_LEN(token));
+                }
                 else
                 {
                     // peek ahead to see next token
@@ -2117,7 +2206,10 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
 
             case IMG_START:
                 if (IN(NO_WIKI_START) || IN(PRE) || IN(PRE_START))
+                {
+                    _Wikitext_emit_pending_crlf_if_necessary(parser);
                     rb_str_cat(parser->output, token->start, TOKEN_LEN(token));
+                }
                 else if (!NIL_P(parser->capture))
                     rb_str_cat(parser->capture, token->start, TOKEN_LEN(token));
                 else
@@ -2126,39 +2218,38 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
                     _Wikitext_pop_excess_elements(parser);
                     _Wikitext_start_para_if_necessary(parser);
 
-                    // peek ahead to see next token
-                    NEXT_TOKEN();
-                    if (token->type != PRINTABLE)
-                        // failure
-                        rb_str_cat(parser->output, literal_img_start, sizeof(literal_img_start) - 1);
-                    else
+                    // scan ahead consuming PRINTABLE and SPECIAL_URI_CHARS tokens
+                    // will cheat here and abuse the link_target capture buffer to accumulate text
+                    if (NIL_P(parser->link_target))
+                        parser->link_target = rb_str_new2("");
+                    while (NEXT_TOKEN(), (type = token->type))
                     {
-                        // remember the PRINTABLE
-                        char    *token_ptr  = token->start;
-                        int     token_len   = TOKEN_LEN(token);
-
-                        // peek ahead once more
-                        NEXT_TOKEN();
-                        if (token->type == IMG_END)
+                        if (type == PRINTABLE || type == SPECIAL_URI_CHARS)
+                            rb_str_cat(parser->link_target, token->start, TOKEN_LEN(token));
+                        else if (type == IMG_END)
                         {
                             // success
-                            _Wikitext_append_img(parser, token_ptr, token_len);
+                            _Wikitext_append_img(parser, RSTRING_PTR(parser->link_target), RSTRING_LEN(parser->link_target));
                             token = NULL;
+                            break;
                         }
-                        else
+                        else // unexpected token (syntax error)
                         {
-                            // failure
+                            // rollback
                             rb_str_cat(parser->output, literal_img_start, sizeof(literal_img_start) - 1);
-                            rb_str_cat(parser->output, token_ptr, token_len);
+                            rb_str_cat(parser->output, RSTRING_PTR(parser->link_target), RSTRING_LEN(parser->link_target));
+                            break;
                         }
                     }
 
                     // jump to top of the loop to process token we scanned during lookahead
+                    parser->link_target = Qnil;
                     continue;
                 }
                 break;
 
             case CRLF:
+                i = parser->pending_crlf;
                 parser->pending_crlf = Qfalse;
                 _Wikitext_rollback_failed_link(parser);             // if any
                 _Wikitext_rollback_failed_external_link(parser);    // if any
@@ -2170,27 +2261,33 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
                 }
                 else if (IN(PRE))
                 {
-                    // beware when nothing or BLOCKQUOTE on line buffer (not line stack!) prior to CRLF, that must be end of PRE block
-                    if (NO_ITEM(ary_entry(parser->line_buffer, -2)) || ary_entry(parser->line_buffer, -2) == BLOCKQUOTE)
+                    // beware when BLOCKQUOTE on line buffer (not line stack!) prior to CRLF, that must be end of PRE block
+                    if (ary_entry(parser->line_buffer, -2) == BLOCKQUOTE)
                         // don't emit in this case
                         _Wikitext_pop_from_stack_up_to(parser, parser->output, PRE, Qtrue);
                     else
                     {
+                        if (ary_entry(parser->line_buffer, -2) == PRE)
+                        {
+                             // only thing on line is the PRE: emit pending line ending (if we had one)
+                             if (i == Qtrue)
+                                 rb_str_cat(parser->output, parser->line_ending->ptr, parser->line_ending->len);
+                        }
+
+                        // clear these _before_ calling NEXT_TOKEN (NEXT_TOKEN adds to the line_buffer)
+                        ary_clear(parser->line);
+                        ary_clear(parser->line_buffer);
+
                         // peek ahead to see if this is definitely the end of the PRE block
                         NEXT_TOKEN();
                         type = token->type;
                         if (type != BLOCKQUOTE && type != PRE)
-                        {
                             // this is definitely the end of the block, so don't emit
                             _Wikitext_pop_from_stack_up_to(parser, parser->output, PRE, Qtrue);
-                        }
                         else
                             // potentially will emit
                             parser->pending_crlf = Qtrue;
 
-                        // delete the entire contents of the line scope stack and buffer
-                        ary_clear(parser->line);
-                        ary_clear(parser->line_buffer);
                         continue; // jump back to top of loop to handle token grabbed via lookahead
                     }
                 }
@@ -2233,6 +2330,7 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
                 ary_clear(parser->line_buffer);
                 break;
 
+            case SPECIAL_URI_CHARS:
             case PRINTABLE:
             case IMG_END:
             case LEFT_CURLY:
@@ -2251,6 +2349,12 @@ VALUE Wikitext_parser_parse(int argc, VALUE *argv, VALUE self)
                 break;
 
             case END_OF_FILE:
+                // special case for input like " foo\n " (see pre_spec.rb)
+                if (IN(PRE) &&
+                    ary_entry(parser->line_buffer, -2) == PRE &&
+                    parser->pending_crlf == Qtrue)
+                    rb_str_cat(parser->output, parser->line_ending->ptr, parser->line_ending->len);
+
                 // close any open scopes on hitting EOF
                 _Wikitext_rollback_failed_external_link(parser);    // if any
                 _Wikitext_rollback_failed_link(parser);             // if any