breakout_parser 0.0.0

data/ext/breakout_parser/main.c

@@ -0,0 +1,32 @@
+#ifndef RUBY_VERSION
+
+// this file is compiled in ONLY if RUBY_VERSION is NOT defined (compiling w/o ruby)
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+extern char*buf, *bufptr;
+extern char*in_buf;
+extern size_t in_buf_len, bufsize;
+
+main(){
+    char *p;
+//    *buf = 0;
+//    strcpy(in_buf,"h1. aaa");
+//    in_buf_len = strlen(in_buf);
+
+    bufsize    = 1 + in_buf_len + in_buf_len/3; // reserve 30% of in_buf size
+    if(bufsize<0x100) bufsize = 0x100;
+
+    buf = malloc(bufsize); 
+    bufptr = buf;
+
+    yyparse();
+
+    p = buf;
+    while(strncmp(p,"<br />",6) == 0) p += 6;
+    printf("%s\n",p);
+}
+
+#endif // ifndef RUBY_VERSION

data/ext/breakout_parser/make_win32.bat

@@ -0,0 +1,15 @@
+@echo off
+ruby extconf.rb
+if errorlevel 1 goto err
+nmake
+if errorlevel 1 goto err
+mt.exe -manifest breakout_parser.so.manifest -outputresource:breakout_parser.so;2
+
+echo ALL OK!
+goto end
+
+:err
+echo ERROR!
+goto end
+
+:end

data/ext/breakout_parser/parser.l

@@ -0,0 +1,162 @@
+%{
+#include <stdlib.h>
+#include "parser.tab.h"
+
+// HACK for parsing end of bold/italic text at EOF
+// see http://flex.sourceforge.net/manual/How-can-I-match-text-only-at-the-end-of-a-file_003f.html#How-can-I-match-text-only-at-the-end-of-a-file_003f
+int was_fake_br = 0;
+
+extern char* in_buf;
+extern char* in_pos;
+extern size_t in_buf_len;
+
+#undef  YY_BUF_SIZE
+#define YY_BUF_SIZE 1300000
+#undef  YY_READ_BUF_SIZE
+#define YY_READ_BUF_SIZE 1200000
+
+#define YY_NO_UNISTD_H
+
+
+#define MIN(a,b) ((a) < (b)) ? (a) : (b)
+
+#ifdef RUBY_VERSION
+
+#define YY_INPUT(buf,result,max_size) \
+        { \
+            size_t unread_size = in_buf_len - (in_pos - in_buf); \
+            if( unread_size > 0 ){ \
+                size_t blocksize = MIN(max_size, unread_size); \
+                was_fake_br = 0; \
+                memcpy(buf,in_pos,blocksize); \
+                in_pos += blocksize; \
+                result = blocksize; \
+            } else if ( !was_fake_br ){ \
+                memcpy(buf,"\n\xff",2); result = 2; was_fake_br = 1; \
+            } else { \
+                result = 0; \
+            } \
+        }
+
+#else
+
+#define YY_INPUT(buf,result,max_size) \
+        if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \
+                { \
+                int c = '*'; \
+                int n; \
+                for ( n = 0; n < max_size && \
+                             (c = getc( yyin )) != EOF && c != '\n'; ++n ) \
+                        buf[n] = (char) c; \
+                if ( c == '\n' ) \
+                        buf[n++] = (char) c; \
+                if ( c == EOF && ferror( yyin ) ) \
+                        YY_FATAL_ERROR( "input in flex scanner failed" ); \
+                result = n; \
+                } \
+        else \
+                { \
+                errno=0; \
+                while ( (result = fread(buf, 1, max_size, yyin))==0 && ferror(yyin)) \
+                        { \
+                        if( errno != EINTR) \
+                                { \
+                                YY_FATAL_ERROR( "input in flex scanner failed" ); \
+                                break; \
+                                } \
+                        errno=0; \
+                        clearerr(yyin); \
+                        } \
+                }\
+                if( result == 0 && errno == 0 && was_fake_br == 0 ){\
+                        strcpy(buf,"\n\xff"); result = 2; was_fake_br = 1; \
+                }
+
+#endif // ifdef RUBY_VERSION
+
+%}
+
+%option stack
+%x _PRE
+%s _BOLD _ITALIC
+
+
+
+%%
+
+\xff                    ; /* EOF mark. should not appear in valid UTF8 text */
+[\r\n\t ]+\xff          ; /* EOF + skip tailing whitespace */
+
+
+([ \t]*\n){2,}          { return BRBR; }
+([ \t]*\r){2,}          { return BRBR; }
+([ \t]*\r\n){2,}        { return BRBR; }
+
+[ \t]*(\r\n|\r|\n)      { return BR; }
+
+\<pre>[ \t]*<code>[\r\n]*  { BEGIN _PRE; return PRE_START; }
+
+^\*/[^ \t\r\n*.]          { yylval.ivalue=0; yy_push_state(_BOLD); return BOLD_START; }
+[ \t]+\*/[^ \t\r\n*.]     { yylval.ivalue=1; yy_push_state(_BOLD); return BOLD_START; }
+
+^_/[^ \t\r\n_]            { yylval.ivalue=0; yy_push_state(_ITALIC); return ITALIC_START; }
+[ \t]+_/[^ \t\r\n_]       { yylval.ivalue=1; yy_push_state(_ITALIC); return ITALIC_START; }
+
+^h[1-5]\.[ \t]+[^ \t\r\n][^\r\n]*/[\r\n\xff] {
+    yylval.svalue = yytext+4; 
+    switch(yytext[1]){
+        case '1': return H1;
+        case '2': return H2;
+        case '3': return H3;
+        case '4': return H4;
+        case '5': return H5;
+    }
+    return H1; 
+}
+
+^[ \t]*\*[ ]+              { return ULI; }
+^[ \t]*#[ ]+               { return OLI; }
+
+https?:\/\/[^ \r\n<>"(){}*]+[^ \r\n<>"(){}*,.\[\]]  { yylval.svalue = yytext; return URL; }
+
+#[0-9]+/[ \t\r\n,.;()]  { yylval.svalue = yytext; return TICKET_LINK; }
+\[\[ticket:[0-9]+(\|[^\[\]]+)?\]\]   { yylval.svalue = yytext; return TICKET_LINK; }
+
+\[\[revision:[0-9]+(\|[^\[\]]+)?\]\]    { yylval.svalue = yytext+11; return SVN_REVISION_LINK; }
+\[\[revision:[0-9a-f]+(\|[^\[\]]+)?\]\] { yylval.svalue = yytext+11; return GIT_REVISION_LINK; }
+\[\[r:[0-9]+(\|[^\[\]]+)?\]\]           { yylval.svalue = yytext+4;  return SVN_REVISION_LINK; }
+\[\[r:[0-9a-f]+(\|[^\[\]]+)?\]\]        { yylval.svalue = yytext+4;  return GIT_REVISION_LINK; }
+
+\[\[url:[a-z]+:\/\/[^]]+\]\] { yylval.svalue = yytext+6; return URL_WITH_PROTO_LINK; }
+\[\[url:\/[^]]+\]\]          { yylval.svalue = yytext+6; return URL_WITH_PROTO_LINK; }
+\[\[url:[^]]+\]\]            { yylval.svalue = yytext+6; return URL_WITHOUT_PROTO_LINK; }
+
+\[\[wiki:[^]]+\]\]           { yylval.svalue = yytext+7; return WIKI_LINK; }
+\[\[[a-zA-Z0-9_-][a-zA-Z0-9_#-]+(\|[^\[\]]+)?\]\] { yylval.svalue = yytext+2; return WIKI_LINK; }
+
+\[\[#[^\r\n\[\]]+\]\]   { yylval.svalue = yytext+3; return ANCHOR_LINK; }
+
+^[ \t]+                 ; /* skip spaces at line start */
+[ \t]+                  { yylval.ivalue = ' '; return T_CHAR; }
+
+[a-zA-Z0-9]+            { yylval.svalue = yytext; return T_WORD; }
+
+.                       { yylval.ivalue = yytext[0]; return T_CHAR; }
+
+
+
+<_PRE>[ \t\r\n]*<\/code>[ \t]*<\/pre>  { BEGIN INITIAL; return PRE_END; }
+<_PRE>[ \t\r\n]*\xff                   { BEGIN INITIAL; return PRE_END; }
+<_PRE>(?s:.)                 { yylval.ivalue = yytext[0]; return T_CHAR; }
+<_PRE><<EOF>>                { BEGIN INITIAL; return PRE_END; }
+
+<_BOLD>[ \t]+\*/[ \t]+       { yylval.svalue = " *"; return T_WORD; } /* skip lone star */
+<_BOLD>\*/[ \t\r\n'.,]       { yy_pop_state(); return BOLD_END; }
+<_BOLD><<EOF>>               { yy_pop_state(); return BOLD_END; }
+
+<_ITALIC>[ \t]+_/[ \t]+      { yylval.svalue = " _"; return T_WORD; } /* skip lone underscore */
+<_ITALIC>_/[ \t\r\n'.,]      { yy_pop_state(); return ITALIC_END; }
+<_ITALIC><<EOF>>             { yy_pop_state(); return ITALIC_END; }
+
+%%
+

data/ext/breakout_parser/parser.tab.h

@@ -0,0 +1,98 @@
+
+/* A Bison parser, made by GNU Bison 2.4.1.  */
+
+/* Skeleton interface for Bison's Yacc-like parsers in C
+   
+      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
+   Free Software Foundation, Inc.
+   
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* As a special exception, you may create a larger work that contains
+   part or all of the Bison parser skeleton and distribute that work
+   under terms of your choice, so long as that work isn't itself a
+   parser generator using the skeleton or a modified version thereof
+   as a parser skeleton.  Alternatively, if you modify or redistribute
+   the parser skeleton itself, you may (at your option) remove this
+   special exception, which will cause the skeleton and the resulting
+   Bison output files to be licensed under the GNU General Public
+   License without this special exception.
+   
+   This special exception was added by the Free Software Foundation in
+   version 2.2 of Bison.  */
+
+
+/* Tokens.  */
+#ifndef YYTOKENTYPE
+# define YYTOKENTYPE
+   /* Put the tokens into the symbol table, so that GDB and other debuggers
+      know about them.  */
+   enum yytokentype {
+     T_CHAR = 258,
+     BOLD_START = 259,
+     ITALIC_START = 260,
+     T_WORD = 261,
+     TICKET_LINK = 262,
+     LINK = 263,
+     SVN_REVISION_LINK = 264,
+     GIT_REVISION_LINK = 265,
+     WIKI_LINK = 266,
+     ANCHOR_LINK = 267,
+     URL_WITH_PROTO_LINK = 268,
+     URL_WITHOUT_PROTO_LINK = 269,
+     URL = 270,
+     UL = 271,
+     H1 = 272,
+     H2 = 273,
+     H3 = 274,
+     H4 = 275,
+     H5 = 276,
+     SPACE = 277,
+     BR = 278,
+     BRBR = 279,
+     OLI = 280,
+     ULI = 281,
+     PRE_START = 282,
+     PRE_END = 283,
+     BOLD_END = 284,
+     ITALIC_END = 285
+   };
+#endif
+
+
+
+#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
+typedef union YYSTYPE
+{
+
+/* Line 1676 of yacc.c  */
+#line 59 "parser.y"
+
+        double dvalue;
+        int ivalue;
+        char const* svalue;
+
+
+
+/* Line 1676 of yacc.c  */
+#line 90 "parser.tab.h"
+} YYSTYPE;
+# define YYSTYPE_IS_TRIVIAL 1
+# define yystype YYSTYPE /* obsolescent; will be withdrawn */
+# define YYSTYPE_IS_DECLARED 1
+#endif
+
+extern YYSTYPE yylval;
+
+

data/ext/breakout_parser/parser.y

@@ -0,0 +1,357 @@
+%{ 
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef RUBY_VERSION
+#include "ruby.h"
+#else
+#define REALLOC_N(ptr,type,n) ptr=realloc(ptr,sizeof(type)*n)
+#endif
+
+extern int yylex();
+
+char *in_buf = NULL, *in_pos = NULL;
+size_t in_buf_len = 0;
+
+char *buf = NULL, *bufptr = NULL;
+size_t bufsize = 0;
+char *space_name = "test_space";
+
+#define CHECK_BUF_SIZE(len) \
+    if( (bufptr - buf + len + 1) >= bufsize ){ \
+        /*printf("[.] REALLOC oldsz=%d, newsz=%d\n",bufsize, (bufsize+((len > 0x1000) ? (len+0x1000) : 0x1000)));*/ \
+        char *oldbuf = buf; \
+        bufsize += (len > 0x1000) ? (len+0x1000) : 0x1000; \
+        REALLOC_N(buf, char, bufsize); \
+        bufptr = buf + (bufptr-oldbuf); \
+    }
+
+concat(const char*what, size_t len){
+//    printf("[.] concat: w=\"%s\", l=%d\n",what,len);
+    CHECK_BUF_SIZE(len);
+    memcpy(bufptr, what, len);
+    bufptr += len;
+//    printf("[.] concat OK\n");
+}
+
+// it's better to use concat(), especially if "what"'s length is always predefined
+concat2(const char*what){
+    size_t len = strlen(what);
+    concat(what,len);
+}
+
+// better error reporting
+#define YYERROR_VERBOSE
+
+// more stacks
+#define YYMAXDEPTH  0x180000
+
+// bison requires that you supply this function
+void yyerror(const char *msg)
+{
+      printf("ERROR(PARSER): %s\n", msg);
+}
+
+
+%}
+
+%union {
+        double dvalue;
+        int ivalue;
+        char const* svalue;
+}
+
+
+%token <ivalue> T_CHAR BOLD_START ITALIC_START
+%token <svalue> T_WORD TICKET_LINK LINK SVN_REVISION_LINK GIT_REVISION_LINK WIKI_LINK ANCHOR_LINK
+%token <svalue> URL_WITH_PROTO_LINK URL_WITHOUT_PROTO_LINK
+%token <svalue> URL
+%token <svalue> UL
+%token <svalue> H1 H2 H3 H4 H5
+%token SPACE BR BRBR OLI ULI
+%token PRE_START PRE_END 
+%token BOLD_END ITALIC_END
+
+//%type <dvalue> expression
+//%type <dvalue> term
+//%type <dvalue> varornum
+%%
+text : 
+     | textitem text
+
+
+textitem: br
+     | words
+     | h1 {concat("</h1>",5)}
+     | h2 {concat("</h2>",5)}
+     | h3 {concat("</h3>",5)}
+     | h4 {concat("</h4>",5)}
+     | h5 {concat("</h5>",5)}
+     | {concat("<ul>",4)} ulist {concat("</ul>",5)} textitem
+     | {concat("<ol>",4)} olist {concat("</ol>",5)} textitem
+     | pre
+
+ulist: ulitem        {concat("</li>",5)}
+     | ulist ulitem  {concat("</li>",5)}
+
+ulitem: uli words
+      | uli words BR
+
+olist: olitem        {concat("</li>",5)}
+     | olist olitem  {concat("</li>",5)}
+
+olitem: oli words
+      | oli words BR
+
+words: word
+     | word words
+
+word : chars
+      | link
+      | T_WORD           {concat2($1)} // TODO: somehow pass T_WORD's length here
+      | URL            {process_url($1)}
+      | BOLD_START     {$1 ? concat(" <strong>",9) : concat("<strong>",8)}
+      | BOLD_END       {concat("</strong>",9)}
+      | ITALIC_START   {$1 ? concat(" <em>",5) : concat("<em>",4)}
+      | ITALIC_END     {concat("</em>",5)}
+
+link: TICKET_LINK            {process_ticket_link($1)}
+    | SVN_REVISION_LINK      {process_svn_link($1)}
+    | GIT_REVISION_LINK      {process_git_link($1)}
+    | URL_WITH_PROTO_LINK    {process_url_link($1,NULL)}
+    | URL_WITHOUT_PROTO_LINK {process_url_link($1,"http://")}
+    | WIKI_LINK              {process_wiki_link($1)}
+    | ANCHOR_LINK            {process_anchor_link($1)}
+
+chars:
+     | char chars
+
+char : T_CHAR {concat_escaped_char($1)}
+
+//raw_chars:
+//     | raw_char raw_chars
+
+//raw_char : T_CHAR {concat_raw_char($1)}
+
+h1   : H1          {concat("<h1 id=\"",8); process_header($1)}
+h2   : H2          {concat("<h2 id=\"",8); process_header($1)}
+h3   : H3          {concat("<h3 id=\"",8); process_header($1)}
+h4   : H4          {concat("<h4 id=\"",8); process_header($1)}
+h5   : H5          {concat("<h5 id=\"",8); process_header($1)}
+ul   : UL          {concat("<ul>",4)}
+oli  : OLI         {concat("<li>",4)}
+uli  : ULI         {concat("<li>",4)}
+br   : BR          {concat("<br />",6)}
+     | BRBR        {concat("<br /><br />",12)}
+pre  : PRE_START   {concat("<pre><code>",11)} chars PRE_END {concat("</code></pre>",13)}
+
+//word : T_WORD        { process_word($1); }
+
+%%
+
+concat_hex_char(char c){
+    unsigned char d;
+    d = ((unsigned char)c)>>4;
+    concat_raw_char(d>9 ? ('a'+d-10) : '0'+d);
+    d = c&0x0f;
+    concat_raw_char(d>9 ? ('a'+d-10) : '0'+d);
+}
+
+need_hex_convert(const char*p, const char*pend){
+    // scan for non alphanum chars first
+    for(; *p && p<=pend; p++){
+        if( *p == ' ' || 
+            *p == '_' || 
+            *p == '-' || 
+            *p == '.' ||
+            (*p >= '0' && *p <= '9') ||
+            (*p >= 'a' && *p <= 'z') ||
+            (*p >= 'A' && *p <= 'Z')
+        ){
+            // allowed char
+        } else {
+            // forbidden char found
+            return 1;
+        }
+    }
+    return 0;
+}
+
+process_header(const char*title){
+    const char*p,*pend;
+
+    // skip heading spaces
+    while(*title == ' ' || *title == '\t') title++;
+
+    // skip tailing spaces
+    p = title;
+    pend = p+strlen(p)-1;
+    while(pend > p && (*pend == ' ' || *pend == '\t')) pend--;
+
+    // concat 'id'
+    if( need_hex_convert(title,pend) ){
+        for(p = title; *p && p<=pend; p++) concat_hex_char( *p );
+    } else {
+        for(p = title; *p && p<=pend; p++) concat_raw_char( *p == ' ' ? '_' : *p);
+    }
+    concat("\">",2);
+    for(p = title; *p && p<=pend; p++) concat_escaped_char( *p );
+}
+
+process_link_tail(const char*text,const char*pend,const char*prepend){
+    const char*p;
+
+    concat("\">",2);
+
+    if(!pend){
+        pend = strchr(text,']');
+        pend = pend ? (pend-1) : (text + strlen(text) - 1);
+    }
+
+    // find start of TITLE: [[url:http://www.ru|TITLE]]
+    if(p = strchr(text,'|')){
+        // title found
+        p++;
+    } else {
+        // no title
+        if(prepend) concat2(prepend);
+        p = text;
+    }
+
+    for(; *p && p<=pend; p++) concat_escaped_char( *p );
+    concat("</a>",4);
+}
+
+process_anchor_link(const char*target){
+    const char *p,*pend;
+
+    // skip tail
+    p = target;
+    pend = p+strlen(p)-1;
+    while(pend > p && (*pend == ' ' || *pend == '\t' || *pend == ']')) pend--;
+
+    if((p = strchr(target,'|')) && (p<pend) && (p>target)) pend = p-1;
+
+    concat("<a href=\"#",10);
+    if( need_hex_convert(target,pend) ){
+        for(p = target; *p && p<=pend; p++) concat_hex_char( *p );
+    } else {
+        for(p = target; *p && p<=pend; p++) concat_raw_char( *p == ' ' ? '_' : *p);
+    }
+    concat("\" title=\"#",10);
+    for(p = target; *p && p<=pend; p++) concat_raw_char( *p );
+    concat("\" class=\"wiki_link",18);
+    process_link_tail(target,NULL,"#");
+}
+
+process_url_link(const char*target,const char* proto){
+    const char *c;
+    concat("<a href=\"",9);
+    if(proto) concat2(proto);
+    for(c=target; *c && *c != ']' && *c != '|'; c++) concat_raw_char(*c);
+    process_link_tail(target,NULL,proto);
+}
+
+process_svn_link(const char*target){
+    const char *c;
+    concat("<a href=\"http://code.assembla.com/",34);
+    concat2(space_name);
+    concat("/svn/changesets/",16);
+    for(c=target; *c && *c!=']' && *c!='|'; c++) concat_raw_char(*c);
+    process_link_tail(target,NULL,"revision:");
+}
+
+process_git_link(const char*target){
+    const char *c;
+    concat("<a href=\"http://code.assembla.com/",34);
+    concat2(space_name);
+    concat("/git/changesets/",16);
+    for(c=target; *c && *c!=']' && *c!='|'; c++) concat_raw_char(*c);
+    process_link_tail(target,NULL,"revision:");
+}
+
+process_wiki_link(const char*target){
+    const char *c;
+    concat("<a class=\"wiki_link\" title=\"",28);
+    for(c=target; *c && *c!=']' && *c!='|'; c++) concat_raw_char(*c);
+    concat("\" href=\"/wiki/show/",19);
+    concat2(space_name);
+    concat_raw_char('/');
+    for(c=target; *c && *c!=']' && *c!='|'; c++) concat_raw_char(*c);
+    process_link_tail(target,NULL,NULL);
+}
+
+process_ticket_link(const char*ticket_id){
+    const char *c;
+    while(*ticket_id && (*ticket_id < '0' || *ticket_id > '9') ) ticket_id++;
+    concat("<a href=\"/spaces/",17);
+    concat2(space_name);
+    concat("/tickets/",9);
+    for(c=ticket_id; *c && *c>='0' && *c<='9'; c++) concat_raw_char(*c);
+    process_link_tail(ticket_id,NULL,"#");
+}
+
+concat_escaped_char(int c){
+    switch(c){
+        case '<':
+            concat("&lt;",4);
+            break;
+        case '>':
+            concat("&gt;",4);
+            break;
+        case '"':
+            concat("&quot;",6);
+            break;
+        case '&':
+            concat("&amp;",5);
+            break;
+        default:
+            concat_raw_char(c);
+            break;
+    }
+}
+
+
+concat_raw_char(int c){
+    CHECK_BUF_SIZE(1);
+    *bufptr++ = c;
+}
+
+/*
+concat_escaping_html(const char*what){
+    char *po = buf + strlen(buf);
+    const char *pi = what;
+
+    for(;*pi;pi++){
+        switch(*pi){
+            case '<':
+                strcpy(po,"&lt;");
+                po += 4;
+                break;
+            case '>':
+                strcpy(po,"&gt;");
+                po += 4;
+                break;
+            case '"':
+                strcpy(po,"&quot;");
+                po += 6;
+                break;
+            case '&':
+                strcpy(po,"&amp;");
+                po += 5;
+                break;
+            default:
+                *po++ = *pi;
+        }
+    }
+    *po = 0;
+}*/
+
+process_url(const char*url){
+    const char *p;
+
+    concat("<a href=\"",9);
+    for(p=url; *p; p++) concat_raw_char(*p);
+    process_link_tail(url,NULL,NULL);
+}