mediacloth 0.0.2 → 0.0.3

data/lib/mediacloth/mediawikiparser.y

@@ -8,12 +8,15 @@
 # parser.parse(input)
 class MediaWikiParser
 
-token BOLDSTART BOLDEND ITALICSTART ITALICEND LINKSTART LINKEND
-    INTLINKSTART INTLINKEND SECTION_START SECTION_END TEXT PRE
+token BOLDSTART BOLDEND ITALICSTART ITALICEND LINKSTART LINKEND LINKSEP
+    INTLINKSTART INTLINKEND INTLINKSEP RESOURCE_SEP
+    SECTION_START SECTION_END TEXT PRE
     HLINE SIGNATURE_NAME SIGNATURE_DATE SIGNATURE_FULL
     UL_START UL_END LI_START LI_END OL_START OL_END
+    TABLE_START TABLE_END ROW_START ROW_END HEAD_START HEAD_END CELL_START CELL_END
     PARA_START PARA_END
 
+
 rule
 
 wiki:
@@ -47,10 +50,7 @@ contents:
         }
     | section
         {
-            s = SectionAST.new
-            s.contents = val[0][0]
-            s.level = val[0][1]
-            result = s
+            result = val[0]
         }
     | PARA_START para_contents PARA_END
         {
@@ -60,6 +60,29 @@ contents:
                 result = p
             end
         }
+    | LINKSTART link_contents LINKEND
+        {
+            l = LinkAST.new
+            l.url = val[1][0]
+            l.children += val[1][1..-1] if val[1].length > 1
+            result = l
+        }
+    | INTLINKSTART TEXT RESOURCE_SEP TEXT reslink_repeated_contents INTLINKEND
+        {
+            l = ResourceLinkAST.new
+            l.prefix = val[1]
+            l.locator = val[3]
+            l.children = val[4] unless val[4].nil? or val[4].empty?
+            result = l
+        }
+    | INTLINKSTART TEXT intlink_repeated_contents INTLINKEND
+        {
+            l = InternalLinkAST.new
+            l.locator = val[1]
+            l.children = val[2] unless val[2].nil? or val[2].empty?
+            result = l
+        }
+    | table
     ;
 
 #TODO: remove empty paragraphs in lexer
@@ -71,6 +94,60 @@ para_contents:
         {
             result = val[0]
         }
+    ;
+
+link_contents:
+      TEXT
+        {
+            result = val
+        }
+    | TEXT LINKSEP link_repeated_contents
+        {
+            result = [val[0]]
+            result += val[2]
+        }
+    ;
+
+
+link_repeated_contents:
+      repeated_contents
+        {
+            result = val[0]
+        }
+    | repeated_contents LINKSEP link_repeated_contents
+        {
+            result = val[0]
+            result += val[2] if val[2]
+        }
+    ;
+
+
+intlink_repeated_contents:
+        {
+            result = nil
+        }
+    | INTLINKSEP repeated_contents
+        {
+            result = val[1]
+        }
+    ;
+
+reslink_repeated_contents:
+        {
+            result = nil
+        }
+    | INTLINKSEP reslink_repeated_contents
+        {
+            result = val[1]
+        }
+    | INTLINKSEP repeated_contents reslink_repeated_contents
+        {
+            i = InternalLinkItemAST.new
+            i.children = val[1]
+            result = [i]
+            result += val[2] if val[2]
+        }
+    ;
 
 repeated_contents: contents
         {
@@ -98,11 +175,79 @@ text: element
         }
     ;
 
-element: LINKSTART TEXT LINKEND
-        { return [:Link, val[1]] }
-    | INTLINKSTART TEXT INTLINKEND
-        { return [:InternalLink, val[1]] }
-    | TEXT
+table:
+      TABLE_START table_contents TABLE_END
+        {
+            table = TableAST.new
+            table.children = val[1] unless val[1].nil? or val[1].empty?
+            result = table
+        }
+    | TABLE_START TEXT table_contents TABLE_END
+        {
+            table = TableAST.new
+            table.options = val[1]
+            table.children = val[2] unless val[2].nil? or val[2].empty?
+            result = table
+        }
+
+table_contents:
+        {
+            result = nil
+        }
+    | ROW_START row_contents ROW_END table_contents
+        {
+            row = TableRowAST.new
+            row.children = val[1] unless val[1].nil? or val[1].empty?
+            result = [row]
+            result += val[3] unless val[3].nil? or val[3].empty?
+        }
+    | ROW_START TEXT row_contents ROW_END table_contents
+        {
+            row = TableRowAST.new
+            row.children = val[2] unless val[2].nil? or val[2].empty?
+            row.options = val[1]
+            result = [row]
+            result += val[4] unless val[4].nil? or val[4].empty?
+        }
+
+row_contents:
+        {
+            result = nil
+        }
+    | HEAD_START HEAD_END row_contents
+        {
+            cell = TableCellAST.new
+            cell.type = :head
+            result = [cell]
+            result += val[2] unless val[2].nil? or val[2].empty?
+        }
+    | HEAD_START repeated_contents HEAD_END row_contents
+        {
+            cell = TableCellAST.new
+            cell.children = val[1] unless val[1].nil? or val[1].empty?
+            cell.type = :head
+            result = [cell]
+            result += val[3] unless val[3].nil? or val[3].empty?
+        }
+    | CELL_START CELL_END row_contents
+        {
+            cell = TableCellAST.new
+            cell.type = :body
+            result = [cell]
+            result += val[2] unless val[2].nil? or val[2].empty?
+        }
+    | CELL_START repeated_contents CELL_END row_contents
+        {
+            cell = TableCellAST.new
+            cell.children = val[1] unless val[1].nil? or val[1].empty?
+            cell.type = :body
+            result = [cell]
+            result += val[3] unless val[3].nil? or val[3].empty?
+        }
+    
+
+element:
+      TEXT
         { return [:None, val[0]] }
     | HLINE
         { return [:HLine, val[0]] }
@@ -114,7 +259,20 @@ element: LINKSTART TEXT LINKEND
         { return [:SignatureFull, val[0]] }
     ;
 
-formatted_element: BOLDSTART repeated_contents BOLDEND
+formatted_element: 
+      BOLDSTART BOLDEND
+        {
+            result = FormattedAST.new
+            result.formatting = :Bold
+            result
+        } 
+    | ITALICSTART ITALICEND
+        {
+            result = FormattedAST.new
+            result.formatting = :Italic
+            result
+        }
+    | BOLDSTART repeated_contents BOLDEND
         {
             p = FormattedAST.new
             p.formatting = :Bold
@@ -161,7 +319,12 @@ list_contents:
         { result = [] }
     ;
 
-list_item: LI_START repeated_contents LI_END
+list_item: 
+      LI_START LI_END
+        {
+            result = ListItemAST.new
+        }
+    | LI_START repeated_contents LI_END
         {
             li = ListItemAST.new
             li.children += val[1]
@@ -173,8 +336,13 @@ preformatted: PRE
         { result = val[0] }
     ;
 
-section: SECTION_START TEXT SECTION_END
-        { result = [val[1], val[0].length] }
+section: SECTION_START repeated_contents SECTION_END
+        { result = [val[1], val[0].length] 
+            s = SectionAST.new
+            s.children = val[1]
+            s.level = val[0].length
+            result = s
+        }
     ;
 
 end

data/lib/mediacloth/mediawikiparser.y~

@@ -0,0 +1,210 @@
+#The parser for the MediaWiki language.
+#
+#Usage together with a lexer:
+# inputFile = File.new("data/input1", "r")
+# input = inputFile.read
+# parser = MediaWikiParser.new
+# parser.lexer = MediaWikiLexer.new
+# parser.parse(input)
+class MediaWikiParser
+
+token BOLDSTART BOLDEND ITALICSTART ITALICEND LINKSTART LINKEND
+    INTLINKSTART INTLINKEND SECTION_START SECTION_END TEXT PRE
+    HLINE SIGNATURE_NAME SIGNATURE_DATE SIGNATURE_FULL
+    UL_START UL_END LI_START LI_END OL_START OL_END
+    PARA_START PARA_END
+
+rule
+
+wiki:
+    repeated_contents
+        {
+            @nodes.push WikiAST.new
+            #@nodes.last.children.insert(0, val[0])
+            #puts val[0]
+            @nodes.last.children += val[0]
+        }
+    ;
+
+contents:
+      text
+        {
+            result = val[0]
+        }
+    | bulleted_list
+        {
+            result = val[0]
+        }
+    | numbered_list
+        {
+            result = val[0]
+        }
+    | preformatted
+        {
+            p = PreformattedAST.new
+            p.contents = val[0]
+            result = p
+        }
+    | section
+        {
+            s = SectionAST.new
+            s.contents = val[0][0]
+            s.level = val[0][1]
+            result = s
+        }
+    | PARA_START para_contents PARA_END
+        {
+            if val[1]
+                p = ParagraphAST.new
+                p.children = val[1]
+                result = p
+            end
+        }
+    | error
+        {
+            puts "ERR"
+            yyerrok
+        }
+    ;
+
+#TODO: remove empty paragraphs in lexer
+para_contents:
+        {
+            result = nil
+        }
+    | repeated_contents
+        {
+            result = val[0]
+        }
+
+repeated_contents: contents
+        {
+            result = []
+            result << val[0]
+        }
+    | repeated_contents contents
+        {
+            result = []
+            result += val[0]
+            result << val[1]
+        }
+    ;
+
+text: element
+        {
+            p = TextAST.new
+            p.formatting = val[0][0]
+            p.contents = val[0][1]
+            result = p
+        }
+    | formatted_element
+        {
+            result = val[0]
+        }
+    ;
+
+element: LINKSTART TEXT LINKEND
+        { return [:Link, val[1]] }
+    | INTLINKSTART TEXT INTLINKEND
+        { return [:InternalLink, val[1]] }
+    | TEXT
+        { return [:None, val[0]] }
+    | HLINE
+        { return [:HLine, val[0]] }
+    | SIGNATURE_DATE
+        { return [:SignatureDate, val[0]] }
+    | SIGNATURE_NAME
+        { return [:SignatureName, val[0]] }
+    | SIGNATURE_FULL
+        { return [:SignatureFull, val[0]] }
+    ;
+
+formatted_element: BOLDSTART repeated_contents BOLDEND
+        {
+            p = FormattedAST.new
+            p.formatting = :Bold
+            p.children += val[1]
+            result = p
+        }
+    | ITALICSTART repeated_contents ITALICEND
+        {
+            p = FormattedAST.new
+            p.formatting = :Italic
+            p.children += val[1]
+            result = p
+        }
+    ;
+
+bulleted_list: UL_START list_item list_contents UL_END
+        {
+            list = ListAST.new
+            list.list_type = :Bulleted
+            list.children << val[1]
+            list.children += val[2]
+            result = list
+        }
+    ;
+
+numbered_list: OL_START list_item list_contents OL_END
+        {
+            list = ListAST.new
+            list.list_type = :Numbered
+            list.children << val[1]
+            list.children += val[2]
+            result = list
+        }
+    ;
+
+list_contents:
+        { result = [] }
+    list_item list_contents
+        {
+            result << val[1]
+            result += val[2]
+        }
+    |
+        { result = [] }
+    ;
+
+list_item: LI_START repeated_contents LI_END
+        {
+            li = ListItemAST.new
+            li.children += val[1]
+            result = li
+        }
+    ;
+
+preformatted: PRE
+        { result = val[0] }
+    ;
+
+section: SECTION_START TEXT SECTION_END
+        { result = [val[1], val[0].length] }
+    ;
+
+end
+
+---- header ----
+require 'mediacloth/mediawikiast'
+
+---- inner ----
+
+attr_accessor :lexer
+
+def initialize
+    @nodes = []
+    super
+end
+
+#Tokenizes input string and parses it.
+def parse(input)
+    @yydebug=true
+    lexer.tokenize(input)
+    do_parse
+    return @nodes.last
+end
+
+#Asks the lexer to return the next token.
+def next_token
+    return @lexer.lex
+end

data/lib/mediacloth/mediawikiwalker.rb

@@ -21,13 +21,21 @@ protected
 
     #Reimplement this
     def parse_wiki_ast(ast)
-        ast.children.each do |c|
-            parse_formatted(c) if c.class == FormattedAST
-            parse_text(c) if c.class == TextAST
-            parse_list(c) if c.class == ListAST
-            parse_preformatted(c) if c.class == PreformattedAST
-            parse_section(c) if c.class == SectionAST
-            parse_paragraph(c) if c.class == ParagraphAST
+        ast.children.map do |c|
+            r = parse_formatted(c) if c.class == FormattedAST
+            r = parse_text(c) if c.class == TextAST
+            r = parse_list(c) if c.class == ListAST
+            r = parse_preformatted(c) if c.class == PreformattedAST
+            r = parse_section(c) if c.class == SectionAST
+            r = parse_paragraph(c) if c.class == ParagraphAST
+            r = parse_link(c) if c.class == LinkAST
+            r = parse_internal_link(c) if c.class == InternalLinkAST
+            r = parse_resource_link(c) if c.class == ResourceLinkAST
+            r = parse_internal_link_item(c) if c.class == InternalLinkItemAST
+            r = parse_table(c) if c.class == TableAST
+            r = parse_table_row(c) if c.class == TableRowAST
+            r = parse_table_cell(c) if c.class == TableCellAST
+            r
         end
     end
 
@@ -47,7 +55,7 @@ protected
 
     #Reimplement this
     def parse_list(ast)
-        ast.children.each do |c|
+        ast.children.map do |c|
             parse_list_item(c) if c.class == ListItemAST
         end
     end
@@ -63,6 +71,46 @@ protected
 
     #Reimplement this
     def parse_section(ast)
+        parse_wiki_ast(ast)
+    end
+    
+    #Reimplement this
+    def parse_link(ast)
+        parse_wiki_ast(ast)
+    end
+    
+    #Reimplement this
+    def parse_internal_link(ast)
+        ast.children.map do |c|
+            parse_internal_link_item(c) if c.class == InternalLinkItemAST
+        end
+    end
+     
+    #Reimplement this
+    def parse_resource_link(ast)
+        ast.children.map do |c|
+            parse_internal_link_item(c) if c.class == InternalLinkItemAST
+        end
+    end
+
+    #Reimplement this
+    def parse_internal_link_item(ast)
+        parse_wiki_ast(ast)
+    end
+
+    #Reimplement this
+    def parse_table(ast)
+        parse_wiki_ast(ast)
+    end
+
+    #Reimplement this
+    def parse_table_row(ast)
+        parse_wiki_ast(ast)
+    end
+
+    #Reimplement this
+    def parse_table_cell(ast)
+        parse_wiki_ast(ast)
     end
 
 end

data/test/data/html1

@@ -1,6 +1,6 @@
 <p>This is a simple text with <b>Bold text</b> and <i>Italic text</i> inside.
 One paragraph can be written in several lines.</p><p>Another paragraph starts after a blank line.</p><p>
-Another one.</p><p>This is text with Internal Link and <a href="http://www.example.com" rel="nofollow">external link</a>.</p><p>We can have headlines:</p><h1>Headline1</h1><h2>Headline2</h2><h3>Headline3</h3><h4>Headline4</h4><h5>Headline5</h5><h6>Headline6</h6><h7>Headline7</h7><hr></hr><p>
+Another one.</p><p>This is text with <a href="javascript:void(0)">Internal Link</a> and <a href="http://www.example.com">external link</a>.</p><p>We can have headlines:</p><h1> Headline1 </h1><h2> Headline2 </h2><h3> Headline3 </h3><h4> Headline4 </h4><h5> Headline5 </h5><h6> Headline6 </h6><h7> Headline7 </h7><p>Headlines may have formatting:</p><h1> See <a href="javascript:void(0)">Internal Link</a> for more info </h1><h2> This is an <b>important</b> heading </h2><hr></hr><p>
 This is a text after the line.</p><ul><li>foo
 </li><li>foo2
 </li></ul>

data/test/data/html10

@@ -0,0 +1,98 @@
+<p>Some examples of tables.</p><table></table>
+<table><tr></tr>
+</table>
+<table><tr><td> a
+</td><td> b
+</td></tr>
+</table>
+<table><tr><td> a
+</td><td> b
+</td></tr>
+<tr><td> 1
+</td><td> 2
+</td></tr>
+</table>
+<table><tr><td> a </td><td> b
+</td></tr>
+<tr><td> 1 </td><td> 2
+</td></tr>
+</table>
+<table><tr><th> a
+</th><th> b
+</th></tr>
+<tr><td> 1
+</td><td> 2
+</td></tr>
+</table>
+<table><tr><th> a </th><th> b
+</th></tr>
+<tr><td> 1 </td><td> 2
+</td></tr>
+</table>
+<table><tr><td> a
+</td></tr>
+<tr><td> 1
+</td><td> 2
+</td></tr>
+</table>
+<table><tr><td> a
+</td><td> b
+</td></tr>
+<tr><td> 1
+</td></tr>
+</table>
+<table><tr><td> a
+</td><td> b
+</td></tr>
+<tr><td></td><td> 2
+</td></tr>
+</table>
+<table><tr><td> <a href="http://example.com">Example</a></td><td> <b>bold</b></td></tr>
+<tr><td> <a href="javascript:void(0)">Example</a></td><td> <a href="javascript:void(0)">image:example.jpg(1, 2, 3)</a></td></tr>
+</table>
+<table><tr><td> <a href="http://example.com">Example
+</a></td><td> <b>bold
+</b></td></tr>
+<tr><td> <a href="javascript:void(0)">Example
+</a></td><td> <b>bold <i>italic
+</i></b></td></tr>
+</table>
+<table><tr><td> a
+</td><td> b
+</td></tr>
+<tr><td> A list in a cell:
+<ul><li>one
+</li><li>two
+</li></ul></td></tr>
+</table>
+<table><tr><td> a
+</td><td> b
+</td></tr>
+<tr><td> A table in a cell:
+<table><tr><td> 1
+</td><td> 2
+</td></tr>
+<tr><td> one
+</td><td> two
+</td></tr>
+</table>
+</td></tr>
+</table>
+<table border="1"><tr><th> a
+</th><th> b
+</th></tr>
+<tr align="left"><td> 1
+</td><td> 2
+</td></tr>
+</table>
+<p>
+Text before
+</p><table><tr><td> a
+</td><td> b
+</td></tr>
+</table>
+<p>Text before with <i>italic
+</i></p><table><tr><td> a
+</td><td> b
+</td></tr>
+</table>

data/test/data/html3

@@ -1 +1 @@
-<p>--Sat Jan 01 01:01:01 EET 2000CreatorCreator Sat Jan 01 01:01:01 EET 2000</p>
+<p>--Sat Jan 01 01:01:01 UTC 2000CreatorCreator Sat Jan 01 01:01:01 UTC 2000</p>

data/test/data/html4

@@ -1 +1,11 @@
-<p><b><i>Foo</i></b></p>
+<p>Test mixing of bold and italic formatting:</p><p><i>italic<b>bold</b>italic</i></p><p><b>bold<i>italic</i>bold</b></p><p><i><b>boldalic</b></i></p><p><i>italic<b>bold</b>italic</i><b>bold<i>italic</i>bold</b></p><p>
+Test bold and italic wrapped around inline links:</p><p><i><a href="http://example.com">http://example.com</a></i></p><p><b><a href="http://example.com">http://example.com</a></b></p><p><i><a href="http://example.com'">http://example.com'</a></i></p><p><b><a href="http://example.com'">http://example.com'</a> is good</b></p><p><i><a href="http://example.com'">http://example.com'</a> is good</i></p><p>
+Test unclosed bold and italic formatting:</p><p><i>Some italic and now </i>bold
+</p><h2>Heading</h2><p>
+Text</p><p><i>Some italic and now <b>bold
+</b></i></p><h2>Heading</h2><p>
+Text</p><p><i>Some italic and now <b>bold</b></i></p><h2>Heading</h2><p>
+Text</p><p><i>Some italic and now <b>bold
+</b></i></p><ul><li>one
+</li><li>two
+</li></ul><p><i>Some italic and now <b>bold</b></i></p><p>Text</p>

data/test/data/html5

@@ -11,4 +11,8 @@
 <ol><li>a
 </li></ol></li></ol></li></ol></li><li>a
 <ol><li>aa
-</li></ol></li></ol>
+</li></ol></li></ol><ul><li>A
+</li><li>B
+</li><li></li></ul><ol><li>A
+</li><li>B
+</li><li></li></ol>

data/test/data/html7

@@ -1,2 +1 @@
-<p><a href="http://www.example.com" rel="nofollow">external link</a><a href="http://www.example.com" rel="nofollow">http://www.example.com</a> [foo]
-</p>
+<p><a href="http://sun.com">http://sun.com</a></p><p><a href="http://sun.com">http://sun.com</a></p><p><a href="mailto:joe@sun.com">mailto:joe@sun.com</a></p><p><a href="http://sun.com">stars</a></p><p><a href="http://sun.com">stars and moon</a></p><p><a href="http://sun.com">stars and <i>moon</i>and <b>trees</b>and birds</a></p><p><a href="javascript:void(0)">sun</a></p><p><a href="javascript:void(0)">All about Sun</a></p><p><a href="javascript:void(0)">image:sun(All about Sun)</a></p><p><a href="javascript:void(0)">nofollow|All about Sun</a></p><p><a href="javascript:void(0)">image:sun(nofollow, All about Sun)</a></p><p><a href="javascript:void(0)">image:sun(All about <a href="javascript:void(0)">Sun</a>)</a></p><p><a href="javascript:void(0)">image:sun(All about <a href="javascript:void(0)">More about</a>)</a></p><p><a href="javascript:void(0)">image:sun(one, two, three)</a></p><p>[]</p><p>[ ]</p><p>[[]]</p><p>[[ ]]</p>

data/test/data/html8

@@ -1 +1 @@
-<p>=== foo ===</p>
+<h3> foo </h3>