mediacloth 0.0.1

data/lib/mediacloth/mediawikiparser.y~

@@ -0,0 +1,172 @@
+#The parser for the MediaWiki language.
+#
+#Usage together with a lexer:
+# inputFile = File.new("data/input1", "r")
+# input = inputFile.read
+# parser = MediaWikiParser.new
+# parser.lexer = MediaWikiLexer.new
+# parser.parse(input)
+class MediaWikiParser
+
+token BOLDSTART BOLDEND ITALICSTART ITALICEND LINKSTART LINKEND
+    INTLINKSTART INTLINKEND SECTION TEXT PRE
+    HLINE SIGNATURE_NAME SIGNATURE_DATE SIGNATURE_FULL
+    UL_START UL_END LI_START LI_END OL_START OL_END
+
+rule
+
+wiki:
+    repeated_contents
+        {
+            @nodes.push WikiAST.new
+            #@nodes.last.children.insert(0, val[0])
+            #puts val[0]
+            @nodes.last.children += val[0]
+        }
+    ;
+
+contents:
+      text
+        {
+            result = val[0]
+        }
+    | bulleted_list
+        {
+            result = val[0]
+        }
+    | preformatted
+        {
+            p = PreformattedAST.new
+            p.contents = val[0]
+            result = p
+        }
+    | section
+        {
+            s = SectionAST.new
+            s.contents = val[0][0]
+            s.level = val[0][1]
+            result = s
+        }
+    ;
+
+repeated_contents: contents
+        {
+            result = []
+            result << val[0]
+        }
+    | repeated_contents contents
+        {
+            result = []
+            result += val[0]
+            result << val[1]
+        }
+    ;
+
+text: element
+        {
+            p = TextAST.new
+            p.formatting = val[0][0]
+            p.contents = val[0][1]
+            result = p
+        }
+    | formatted_element
+        {
+            result = val[0]
+        }
+    ;
+
+element: LINKSTART TEXT LINKEND
+        { return [:Link, val[1]] }
+    | INTLINKSTART TEXT INTLINKEND
+        { return [:InternalLink, val[1]] }
+    | TEXT
+        { return [:None, val[0]] }
+    | HLINE
+        { return [:HLine, val[0]] }
+    | SIGNATURE_DATE
+        { return [:SignatureDate, val[0]] }
+    | SIGNATURE_NAME
+        { return [:SignatureName, val[0]] }
+    | SIGNATURE_FULL
+        { return [:SignatureFull, val[0]] }
+    ;
+
+formatted_element: BOLDSTART repeated_contents BOLDEND
+        {
+            p = FormattedAST.new
+            p.formatting = :Bold
+            p.children += val[1]
+            result = p
+        }
+    | ITALICSTART repeated_contents ITALICEND
+        {
+            p = FormattedAST.new
+            p.formatting = :Italic
+            p.children += val[1]
+            result = p
+        }
+    ;
+
+bulleted_list: UL_START list_item list_contents UL_END
+        {
+            list = ListAST.new
+            list.type = :Bulleted
+            list.children << val[1]
+            list.children += val[2]
+            result = list
+        }
+    ;
+
+list_contents:
+        { result = [] }
+    list_item list_contents
+        {
+            result << val[1]
+            result += val[2]
+        }
+    |
+        { result = [] }
+    ;
+
+list_item: LI_START repeated_contents LI_END
+        {
+            li = ListItemAST.new
+            li.children += val[1]
+            result = li
+        }
+    ;
+
+preformatted: PRE
+        { result = val[0] }
+    ;
+
+section: SECTION TEXT SECTION
+        { result = [val[1], val[0].length] }
+    ;
+
+end
+
+---- header ----
+require 'mediawikiast'
+
+---- inner ----
+
+attr_accessor :lexer
+
+def initialize
+    @nodes = []
+    super
+end
+
+#Tokenizes input string and parses it.
+def parse(input)
+    @yydebug=true
+    lexer.tokenize(input)
+    do_parse
+    return @nodes.last
+end
+
+#Asks the lexer to return the next token.
+def next_token
+    return @lexer.lex
+end

data/lib/mediacloth/mediawikiwalker.rb

@@ -0,0 +1,62 @@
+require 'mediacloth/mediawikiast'
+
+#Default walker to traverse the parse tree.
+#
+#The walker traverses the entire parse tree and does nothing.
+#To implement some functionality during this process, reimplement
+#<i>parse...</i> methods and don't forget to call super() to not
+#break the walk.
+#
+#Current implementations: MediaWikiHTMLGenerator, DebugWalker
+class MediaWikiWalker
+
+    #Walks through the AST
+    def parse(ast)
+        parse_wiki_ast(ast)
+    end
+
+protected
+
+#===== reimplement these methods and don't forget to call super() ====#
+
+    #Reimplement this
+    def parse_wiki_ast(ast)
+        ast.children.each do |c|
+            parse_formatted(c) if c.class == FormattedAST
+            parse_text(c) if c.class == TextAST
+            parse_list(c) if c.class == ListAST
+            parse_preformatted(c) if c.class == PreformattedAST
+            parse_section(c) if c.class == SectionAST
+        end
+    end
+
+    #Reimplement this
+    def parse_formatted(ast)
+        parse_wiki_ast(ast)
+    end
+
+    #Reimplement this
+    def parse_text(ast)
+    end
+
+    #Reimplement this
+    def parse_list(ast)
+        ast.children.each do |c|
+            parse_list_item(c) if c.class == ListItemAST
+        end
+    end
+
+    #Reimplement this
+    def parse_list_item(ast)
+        parse_wiki_ast(ast)
+    end
+
+    #Reimplement this
+    def parse_preformatted(ast)
+    end
+
+    #Reimplement this
+    def parse_section(ast)
+    end
+
+end

data/lib/mediacloth/mediawikiwalker.rb~

@@ -0,0 +1,62 @@
+require 'mediawikiast'
+
+#Default walker to traverse the parse tree.
+#
+#The walker traverses the entire parse tree and does nothing.
+#To implement some functionality during this process, reimplement
+#<i>parse...</i> methods and don't forget to call super() to not
+#break the walk.
+#
+#Current implementations: MediaWikiHTMLGenerator, DebugWalker
+class MediaWikiWalker
+
+    #Walks through the AST
+    def parse(ast)
+        parse_wiki_ast(ast)
+    end
+
+protected
+
+#===== reimplement these methods and don't forget to call super() ====#
+
+    #Reimplement this
+    def parse_wiki_ast(ast)
+        ast.children.each do |c|
+            parse_formatted(c) if c.class == FormattedAST
+            parse_text(c) if c.class == TextAST
+            parse_list(c) if c.class == ListAST
+            parse_preformatted(c) if c.class == PreformattedAST
+            parse_section(c) if c.class == SectionAST
+        end
+    end
+
+    #Reimplement this
+    def parse_formatted(ast)
+        parse_wiki_ast(ast)
+    end
+
+    #Reimplement this
+    def parse_text(ast)
+    end
+
+    #Reimplement this
+    def parse_list(ast)
+        ast.children.each do |c|
+            parse_list_item(c) if c.class == ListItemAST
+        end
+    end
+
+    #Reimplement this
+    def parse_list_item(ast)
+        parse_wiki_ast(ast)
+    end
+
+    #Reimplement this
+    def parse_preformatted(ast)
+    end
+
+    #Reimplement this
+    def parse_section(ast)
+    end
+
+end

data/lib/mediacloth.rb

@@ -0,0 +1,23 @@
+require 'mediacloth/mediawikilexer'
+require 'mediacloth/mediawikiparser'
+require 'mediacloth/mediawikiast'
+require 'mediacloth/mediawikiparams'
+require 'mediacloth/mediawikiwalker'
+require 'mediacloth/mediawikihtmlgenerator'
+
+#Helper module to facilitate MediaCloth usage.
+module MediaCloth
+
+    #Parses wiki formatted +input+ and generates its html representation.
+    def wiki_to_html(input)
+        parser = MediaWikiParser.new
+        parser.lexer = MediaWikiLexer.new
+        ast = parser.parse(input)
+        walker = MediaWikiHTMLGenerator.new
+        walker.parse(ast)
+        walker.html
+    end
+
+    module_function :wiki_to_html
+
+end

data/lib/mediacloth.rb~

@@ -0,0 +1,23 @@
+require 'mediacloth/mediawikilexer'
+require 'mediacloth/mediawikiparser'
+require 'mediacloth/mediawikiast'
+require 'mediacloth/mediawikiparams'
+require 'mediacloth/mediawikiwalker'
+require 'mediacloth/mediawikihtmlgenerator'
+
+#Helper module to facilitate MediaCloth usage.
+module MediaCloth
+
+    
+    def wiki_to_html(input)
+        parser = MediaWikiParser.new
+        parser.lexer = MediaWikiLexer.new
+        ast = parser.parse(input)
+        walker = MediaWikiHTMLGenerator.new
+        walker.parse(ast)
+        walker.html
+    end
+
+    module_function :wiki_to_html
+
+end

data/test/data/html1

@@ -0,0 +1,21 @@
+This is a simple text with <b>Bold text</b> and <i>Italic text</i> inside.
+One paragraph can be written in several lines.
+
+Another paragraph starts after a blank line.
+
+
+Another one.
+
+This is text with Internal Link and <a href="http://www.example.com" rel="nofollow">external link</a>.
+
+We can have headlines:
+
+
+<h1>Headline1</h1><h2>Headline2</h2><h3>Headline3</h3><h4>Headline4</h4><h5>Headline5</h5><h6>Headline6</h6><h7>Headline7</h7>
+
+<hr></hr>
+This is a text after the line.
+
+<ul><li>foo
+</li><li>foo2
+</li></ul>

data/test/data/html2

@@ -0,0 +1,2 @@
+<hr></hr>
+This is a text

data/test/data/html3

@@ -0,0 +1 @@
+--Sat Jan 01 01:01:01 EET 2000CreatorCreator Sat Jan 01 01:01:01 EET 2000

data/test/data/html4

@@ -0,0 +1 @@
+<b><i>Foo</i></b>

data/test/data/html6

@@ -0,0 +1,8 @@
+This is some text with <b>bold</b> and <i>italic</i> formating.
+The list is also here:
+<ul><li>List Item 1
+<ul><li>Sub list item 1
+</li><li>Sub list item 2
+</li></ul></li><li>List Item 2
+</li></ul>The end
+<hr></hr>

data/test/data/html7

@@ -0,0 +1 @@
+<a href="http://www.example.com" rel="nofollow">external link</a><a href="http://www.example.com" rel="nofollow">http://www.example.com</a> [foo]

data/test/data/input1

@@ -0,0 +1,29 @@
+This is a simple text with '''Bold text''' and ''Italic text'' inside.
+One paragraph can be written in several lines.
+
+Another paragraph starts after a blank line.
+
+
+Another one.
+
+This is text with [[Internal Link]] and [http://www.example.com external link].
+
+We can have headlines:
+
+
+= Headline1 =
+== Headline2 ==
+=== Headline3 ===
+==== Headline4 ====
+===== Headline5 =====
+====== Headline6 ======
+======= Headline7 =======
+
+ This is a preformatted ''' ''' << '' '' text
+ yes
+
+----
+This is a text after the line.
+
+*foo
+* foo2

data/test/data/input2

@@ -0,0 +1,2 @@
+----
+This is a text

data/test/data/input3

@@ -0,0 +1,2 @@
+--~~~~~~~~
+~~~~

data/test/data/input4

@@ -0,0 +1 @@
+'''''Foo'''''

data/test/data/input5

@@ -0,0 +1,12 @@
+*A
+**A
+***a
+***b
+****a
+*B
+**b
+
+#a
+##a
+###a
+####a

data/test/data/input6

@@ -0,0 +1,8 @@
+This is some text with '''bold''' and ''italic'' formating.
+The list is also here:
+* List Item 1
+** Sub list item 1
+** Sub list item 2
+* List Item 2
+The end
+----

data/test/data/input7

@@ -0,0 +1,2 @@
+[http://www.example.com external link]
+http://www.example.com [foo]

data/test/data/lex1

@@ -0,0 +1,23 @@
+TEXTThis is a simple text with BOLDSTART'''TEXTBold textBOLDEND'''TEXT and ITALICSTART''TEXTItalic textITALICEND''TEXT inside.
+One paragraph can be written in several lines.
+
+Another paragraph starts after a blank line.
+
+
+Another one.
+
+This is text with INTLINKSTART[[TEXTInternal LinkINTLINKEND]]TEXT and LINKSTART[TEXThttp://www.example.com external linkLINKEND]TEXT.
+
+We can have headlines:
+
+
+SECTION=TEXT Headline1 SECTION=SECTION==TEXT Headline2 SECTION==SECTION===TEXT Headline3 SECTION===SECTION====TEXT Headline4 SECTION====SECTION=====TEXT Headline5 SECTION=====SECTION======TEXT Headline6 SECTION======SECTION=======TEXT Headline7 SECTION=======TEXT
+
+PREThis is a preformatted ''' ''' << '' '' text
+PREyes
+HLINE----TEXT
+This is a text after the line.
+
+UL_STARTLI_STARTTEXTfoo
+LI_ENDLI_STARTTEXTfoo2
+LI_ENDUL_ENDfalsefalse

data/test/data/lex2

@@ -0,0 +1,2 @@
+HLINE----TEXT
+This is a textfalsefalse

data/test/data/lex3

@@ -0,0 +1 @@
+TEXT--SIGNATURE_DATE~~~~~SIGNATURE_NAME~~~SIGNATURE_FULL~~~~falsefalse

data/test/data/lex4

@@ -0,0 +1 @@
+BOLDSTART'''ITALICSTART''TEXTFooITALICEND''BOLDEND'''falsefalse

data/test/data/lex5

@@ -0,0 +1,12 @@
+UL_STARTLI_STARTTEXTA
+UL_STARTLI_STARTTEXTA
+UL_STARTLI_STARTTEXTa
+LI_ENDLI_STARTTEXTb
+UL_STARTLI_STARTTEXTa
+LI_ENDUL_ENDLI_ENDUL_ENDLI_ENDUL_ENDLI_ENDLI_STARTTEXTB
+UL_STARTLI_STARTTEXTb
+LI_ENDUL_ENDLI_ENDUL_ENDOL_STARTLI_STARTTEXTa
+OL_STARTLI_STARTTEXTa
+OL_STARTLI_STARTTEXTa
+OL_STARTLI_STARTTEXTa
+LI_ENDOL_ENDLI_ENDOL_ENDLI_ENDOL_ENDLI_ENDOL_ENDfalsefalse

data/test/data/lex6

@@ -0,0 +1,8 @@
+TEXTThis is some text with BOLDSTART'''TEXTboldBOLDEND'''TEXT and ITALICSTART''TEXTitalicITALICEND''TEXT formating.
+The list is also here:
+UL_STARTLI_STARTTEXTList Item 1
+UL_STARTLI_STARTTEXTSub list item 1
+LI_ENDLI_STARTTEXTSub list item 2
+LI_ENDUL_ENDLI_ENDLI_STARTTEXTList Item 2
+LI_ENDUL_ENDTEXTThe end
+HLINE----falsefalse

data/test/data/lex7

@@ -0,0 +1,2 @@
+LINKSTART[TEXThttp://www.example.com external linkLINKEND]LINKSTARTTEXThttp://www.example.comLINKEND]TEXT [foo]
+falsefalse

data/test/data/result1

@@ -0,0 +1,48 @@
+<p>This is a simple text with <b>Bold text</b> and <i>Italic text</i> inside.
+One paragraph can be written in several lines.
+
+</p><p>Another paragraph starts after a blank line.
+</p><p><br>
+Another one.
+</p><p>This is text with <a href="/wiki/index.php?title=Internal_Link&amp;action=edit" class="new" title="Internal Link">Internal Link</a> and <a href="http://www.example.com" class="external text" title="http://www.example.com" rel="nofollow">external link</a>.
+</p><p>We can have headlines:
+</p><p><br>
+</p>
+<table id="toc" class="toc" summary="Contents"><tbody><tr><td><div id="toctitle"><h2>Contents</h2> <span class="toctoggle">[<a href="javascript:toggleToc()" class="internal" id="togglelink">hide</a>]</span></div>
+
+<ul style="display: block;">
+<li class="toclevel-1"><a href="#Headline1"><span class="tocnumber">1</span> <span class="toctext">Headline1</span></a>
+<ul>
+<li class="toclevel-2"><a href="#Headline2"><span class="tocnumber">1.1</span> <span class="toctext">Headline2</span></a>
+<ul>
+<li class="toclevel-3"><a href="#Headline3"><span class="tocnumber">1.1.1</span> <span class="toctext">Headline3</span></a>
+<ul>
+<li class="toclevel-4"><a href="#Headline4"><span class="tocnumber">1.1.1.1</span> <span class="toctext">Headline4</span></a></li>
+
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</td></tr></tbody></table><script type="text/javascript"> if (window.showTocToggle) { var tocShowText = "show"; var tocHideText = "hide"; showTocToggle(); } </script>
+<div class="editsection" style="float: right; margin-left: 5px;">[<a href="/wiki/index.php?title=Test&amp;action=edit&amp;section=1" title="Edit section: Headline1">edit</a>]</div><a name="Headline1"></a><h1> Headline1 </h1>
+<div class="editsection" style="float: right; margin-left: 5px;">[<a href="/wiki/index.php?title=Test&amp;action=edit&amp;section=2" title="Edit section: Headline2">edit</a>]</div><a name="Headline2"></a><h2> Headline2 </h2>
+
+<div class="editsection" style="float: right; margin-left: 5px;">[<a href="/wiki/index.php?title=Test&amp;action=edit&amp;section=3" title="Edit section: Headline3">edit</a>]</div><a name="Headline3"></a><h3> Headline3 </h3>
+<div class="editsection" style="float: right; margin-left: 5px;">[<a href="/wiki/index.php?title=Test&amp;action=edit&amp;section=4" title="Edit section: Headline4">edit</a>]</div><a name="Headline4"></a><h4> Headline4 </h4>
+<div class="editsection" style="float: right; margin-left: 5px;">[<a href="/wiki/index.php?title=Test&amp;action=edit&amp;section=5" title="Edit section: Headline5">edit</a>]</div><a name="Headline5"></a><h5> Headline5 </h5>
+<div class="editsection" style="float: right; margin-left: 5px;">[<a href="/wiki/index.php?title=Test&amp;action=edit&amp;section=6" title="Edit section: Headline6">edit</a>]</div><a name="Headline6"></a><h6> Headline6 </h6>
+<div class="editsection" style="float: right; margin-left: 5px;">[<a href="/wiki/index.php?title=Test&amp;action=edit&amp;section=7" title="Edit section: = Headline7 =">edit</a>]</div><a name=".3D_Headline7_.3D"></a><h6>= Headline7 =</h6>
+<pre>This is a preformatted <b> </b> &lt;&lt; <i> </i> text
+yes
+
+
+</pre>
+<hr>
+<p>This is a text after the line.
+</p>
+<ul><li>foo
+</li><li> foo2
+</li></ul>

data/test/dataproducers/html.rb

@@ -0,0 +1,23 @@
+require 'mediawikilexer'
+require 'mediawikiparser'
+require 'mediawikiparams'
+require 'mediawikihtmlgenerator'
+
+def produce(index)
+    file = File.new("../data/html#{index}", "w")
+    inputFile = File.new("../data/input#{index}", "r")
+    input = inputFile.read
+
+    parser = MediaWikiParser.new
+    parser.lexer = MediaWikiLexer.new
+    ast = parser.parse(input)
+    MediaWikiParams.instance.time = Time.mktime(2000, 1, 1, 1, 1, 1, 1)
+    generator = MediaWikiHTMLGenerator.new
+    generator.parse(ast)
+
+    file.write(generator.html)
+    file.close
+end
+
+# (3..5).each { |i| produce(i) }
+produce(6)

data/test/dataproducers/html.rb~

@@ -0,0 +1,23 @@
+require 'mediawikilexer'
+require 'mediawikiparser'
+require 'mediawikiparams'
+require 'mediawikihtmlgenerator'
+
+def produce(index)
+    file = File.new("../data/html#{index}", "w")
+    inputFile = File.new("../data/input#{index}", "r")
+    input = inputFile.read
+
+    parser = MediaWikiParser.new
+    parser.lexer = MediaWikiLexer.new
+    ast = parser.parse(input)
+    MediaWikiParams.instance.time = Time.mktime(2000, 1, 1, 1, 1, 1, 1)
+    generator = MediaWikiHTMLGenerator.new
+    generator.parse(ast)
+
+    file.write(generator.html)
+    file.close
+end
+
+# (3..5).each { |i| produce(i) }
+produce(1)

data/test/dataproducers/lex.rb

@@ -0,0 +1,15 @@
+require 'mediawikilexer'
+
+def produce(index)
+    file = File.new("../data/lex#{index}", "w")
+    inputFile = File.new("../data/input#{index}", "r")
+    input = inputFile.read
+
+    lexer = MediaWikiLexer.new
+    tokens = lexer.tokenize(input)
+    file.write(tokens.to_s)
+    file.close
+end
+
+#1..5.each { |i| produce(1) }
+produce(7)