mediacloth 0.0.1 → 0.0.2

data/test/data/html1

@@ -1,21 +1,6 @@
-This is a simple text with <b>Bold text</b> and <i>Italic text</i> inside.
-One paragraph can be written in several lines.
-
-Another paragraph starts after a blank line.
-
-
-Another one.
-
-This is text with Internal Link and <a href="http://www.example.com" rel="nofollow">external link</a>.
-
-We can have headlines:
-
-
-<h1>Headline1</h1><h2>Headline2</h2><h3>Headline3</h3><h4>Headline4</h4><h5>Headline5</h5><h6>Headline6</h6><h7>Headline7</h7>
-
-<hr></hr>
-This is a text after the line.
-
-<ul><li>foo
+<p>This is a simple text with <b>Bold text</b> and <i>Italic text</i> inside.
+One paragraph can be written in several lines.</p><p>Another paragraph starts after a blank line.</p><p>
+Another one.</p><p>This is text with Internal Link and <a href="http://www.example.com" rel="nofollow">external link</a>.</p><p>We can have headlines:</p><h1>Headline1</h1><h2>Headline2</h2><h3>Headline3</h3><h4>Headline4</h4><h5>Headline5</h5><h6>Headline6</h6><h7>Headline7</h7><hr></hr><p>
+This is a text after the line.</p><ul><li>foo
 </li><li>foo2
 </li></ul>

data/test/data/html2

@@ -1,2 +1,2 @@
-<hr></hr>
-This is a text
+<hr></hr><p>
+This is a text</p>

data/test/data/html3

@@ -1 +1 @@
---Sat Jan 01 01:01:01 EET 2000CreatorCreator Sat Jan 01 01:01:01 EET 2000
+<p>--Sat Jan 01 01:01:01 EET 2000CreatorCreator Sat Jan 01 01:01:01 EET 2000</p>

data/test/data/html4

@@ -1 +1 @@
-<b><i>Foo</i></b>
+<p><b><i>Foo</i></b></p>

data/test/data/html5

@@ -0,0 +1,14 @@
+<ul><li>A
+<ul><li>A
+<ul><li>a
+</li><li>b
+<ul><li>a
+</li></ul></li></ul></li></ul></li><li>B
+<ul><li>b
+</li></ul></li></ul><ol><li>a
+<ol><li>a
+<ol><li>a
+<ol><li>a
+</li></ol></li></ol></li></ol></li><li>a
+<ol><li>aa
+</li></ol></li></ol>

data/test/data/html6

@@ -1,8 +1,8 @@
-This is some text with <b>bold</b> and <i>italic</i> formating.
+<p>This is some text with <b>bold</b> and <i>italic</i> formating.
 The list is also here:
-<ul><li>List Item 1
+</p><ul><li>List Item 1
 <ul><li>Sub list item 1
 </li><li>Sub list item 2
 </li></ul></li><li>List Item 2
-</li></ul>The end
-<hr></hr>
+</li></ul><p>The end
+</p><hr></hr>

data/test/data/html7

@@ -1 +1,2 @@
-<a href="http://www.example.com" rel="nofollow">external link</a><a href="http://www.example.com" rel="nofollow">http://www.example.com</a> [foo]
+<p><a href="http://www.example.com" rel="nofollow">external link</a><a href="http://www.example.com" rel="nofollow">http://www.example.com</a> [foo]
+</p>

data/test/data/html8

@@ -0,0 +1 @@
+<p>=== foo ===</p>

data/test/data/input5

@@ -10,3 +10,5 @@
 ##a
 ###a
 ####a
+#a
+##aa

data/test/data/input8

@@ -0,0 +1 @@
+=== foo ===

data/test/data/lex1

@@ -1,22 +1,21 @@
-TEXTThis is a simple text with BOLDSTART'''TEXTBold textBOLDEND'''TEXT and ITALICSTART''TEXTItalic textITALICEND''TEXT inside.
-One paragraph can be written in several lines.
+PARA_STARTTEXTThis is a simple text with BOLDSTART'''TEXTBold textBOLDEND'''TEXT and ITALICSTART''TEXTItalic textITALICEND''TEXT inside.
+One paragraph can be written in several lines.PARA_END
 
-Another paragraph starts after a blank line.
+PARA_STARTTEXTAnother paragraph starts after a blank line.PARA_END
 
+PARA_STARTTEXT
+Another one.PARA_END
 
-Another one.
+PARA_STARTTEXTThis is text with INTLINKSTART[[TEXTInternal LinkINTLINKEND]]TEXT and LINKSTART[TEXThttp://www.example.com external linkLINKEND]TEXT.PARA_END
 
-This is text with INTLINKSTART[[TEXTInternal LinkINTLINKEND]]TEXT and LINKSTART[TEXThttp://www.example.com external linkLINKEND]TEXT.
+PARA_STARTTEXTWe can have headlines:PARA_END
 
-We can have headlines:
-
-
-SECTION=TEXT Headline1 SECTION=SECTION==TEXT Headline2 SECTION==SECTION===TEXT Headline3 SECTION===SECTION====TEXT Headline4 SECTION====SECTION=====TEXT Headline5 SECTION=====SECTION======TEXT Headline6 SECTION======SECTION=======TEXT Headline7 SECTION=======TEXT
+SECTION_START=TEXT Headline1 SECTION_END=SECTION_START==TEXT Headline2 SECTION_END==SECTION_START===TEXT Headline3 SECTION_END===SECTION_START====TEXT Headline4 SECTION_END====SECTION_START=====TEXT Headline5 SECTION_END=====SECTION_START======TEXT Headline6 SECTION_END======SECTION_START=======TEXT Headline7 SECTION_END=======PARA_STARTPARA_END
 
 PREThis is a preformatted ''' ''' << '' '' text
 PREyes
-HLINE----TEXT
-This is a text after the line.
+HLINE----PARA_STARTTEXT
+This is a text after the line.PARA_END
 
 UL_STARTLI_STARTTEXTfoo
 LI_ENDLI_STARTTEXTfoo2

data/test/data/lex2

@@ -1,2 +1,2 @@
-HLINE----TEXT
-This is a textfalsefalse
+HLINE----PARA_STARTTEXT
+This is a textPARA_ENDfalsefalse

data/test/data/lex3

@@ -1 +1 @@
-TEXT--SIGNATURE_DATE~~~~~SIGNATURE_NAME~~~SIGNATURE_FULL~~~~falsefalse
+PARA_STARTTEXT--SIGNATURE_DATE~~~~~SIGNATURE_NAME~~~SIGNATURE_FULL~~~~PARA_ENDfalsefalse

data/test/data/lex4

@@ -1 +1 @@
-BOLDSTART'''ITALICSTART''TEXTFooITALICEND''BOLDEND'''falsefalse
+PARA_STARTBOLDSTART'''ITALICSTART''TEXTFooITALICEND''BOLDEND'''PARA_ENDfalsefalse

data/test/data/lex6

@@ -1,8 +1,8 @@
-TEXTThis is some text with BOLDSTART'''TEXTboldBOLDEND'''TEXT and ITALICSTART''TEXTitalicITALICEND''TEXT formating.
+PARA_STARTTEXTThis is some text with BOLDSTART'''TEXTboldBOLDEND'''TEXT and ITALICSTART''TEXTitalicITALICEND''TEXT formating.
 The list is also here:
-UL_STARTLI_STARTTEXTList Item 1
+PARA_ENDUL_STARTLI_STARTTEXTList Item 1
 UL_STARTLI_STARTTEXTSub list item 1
 LI_ENDLI_STARTTEXTSub list item 2
 LI_ENDUL_ENDLI_ENDLI_STARTTEXTList Item 2
-LI_ENDUL_ENDTEXTThe end
-HLINE----falsefalse
+LI_ENDUL_ENDPARA_STARTTEXTThe end
+PARA_ENDHLINE----falsefalse

data/test/data/lex7

@@ -1,2 +1,2 @@
-LINKSTART[TEXThttp://www.example.com external linkLINKEND]LINKSTARTTEXThttp://www.example.comLINKEND]TEXT [foo]
-falsefalse
+PARA_STARTLINKSTART[TEXThttp://www.example.com external linkLINKEND]LINKSTARTTEXThttp://www.example.comLINKEND]TEXT [foo]
+PARA_ENDfalsefalse

data/test/data/lex8

@@ -0,0 +1 @@
+SECTION_START===TEXT foo SECTION_END===falsefalse

data/test/dataproducers/html.rb

@@ -1,7 +1,7 @@
-require 'mediawikilexer'
-require 'mediawikiparser'
-require 'mediawikiparams'
-require 'mediawikihtmlgenerator'
+require 'mediacloth/mediawikilexer'
+require 'mediacloth/mediawikiparser'
+require 'mediacloth/mediawikiparams'
+require 'mediacloth/mediawikihtmlgenerator'
 
 def produce(index)
     file = File.new("../data/html#{index}", "w")
@@ -15,9 +15,10 @@ def produce(index)
     generator = MediaWikiHTMLGenerator.new
     generator.parse(ast)
 
+#    puts generator.html
     file.write(generator.html)
     file.close
 end
 
 # (3..5).each { |i| produce(i) }
-produce(6)
+produce(5)

data/test/dataproducers/lex.rb

@@ -1,4 +1,4 @@
-require 'mediawikilexer'
+require 'mediacloth/mediawikilexer'
 
 def produce(index)
     file = File.new("../data/lex#{index}", "w")
@@ -12,4 +12,4 @@ def produce(index)
 end
 
 #1..5.each { |i| produce(1) }
-produce(7)
+produce(8)

data/test/debugwalker.rb

@@ -26,6 +26,11 @@ protected
         super(ast)
     end
 
+    def parse_paragraph(ast)
+        info(ast)
+        super(ast)
+    end
+
     def parse_text(ast)
         info(ast, ast.formatting)
         super(ast)

data/test/lexer.rb

@@ -7,13 +7,19 @@ class Lexer_Test < Test::Unit::TestCase
     include TestHelper
 
     def test_input
-        test_files("lex") { |input,result|
+        test_files("lex") { |input,result,resultname|
             lexer = MediaWikiLexer.new
             tokens = lexer.tokenize(input)
             assert_equal(tokens.to_s, result)
         }
     end
 
+    def test_paragraphs
+        assert_equal(lex("Before\n\n\n=Headline="),
+            [[:PARA_START, ""], [:TEXT, "Before"], [:PARA_END, "\n\n"],
+             [:SECTION_START, "="], [:TEXT, "Headline"], [:SECTION_END, "="], [false,false]])
+    end
+
     def test_empty
         assert_equal(lex(""), [[false,false]])
     end
@@ -32,9 +38,12 @@ class Lexer_Test < Test::Unit::TestCase
 
     def test_ending_text_token
         #check for a problem when the last token is TEXT and it's not included
-        assert_equal(lex("\n----\nfoo\n"), [[:HLINE, "----"], [:TEXT, "\nfoo\n"], [false, false]])
+        assert_equal(lex("\n----\nfoo\n"),
+            [[:HLINE, "----"], [:PARA_START, ""],
+                [:TEXT, "\nfoo\n"], [:PARA_END, ""], [false, false]])
         assert_equal(lex("\n----\nfoo\n Hehe"),
-            [[:HLINE, "----"], [:TEXT, "\nfoo\n"], [:PRE, "Hehe"], [false, false]])
+            [[:HLINE, "----"], [:PARA_START, ""], [:TEXT, "\nfoo\n"],
+                [:PARA_END, ""], [:PRE, "Hehe"], [false, false]])
     end
 
     def test_bullets

data/test/testhelper.rb

@@ -19,7 +19,7 @@ module TestHelper
                     result_file = File.new(resultname, "r")
                     result = result_file.read
 
-                    yield(input, result)
+                    yield(input, result, resultname)
                 end
             end
         end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: mediacloth
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
-date: 2006-07-21 00:00:00 +03:00
+  version: 0.0.2
+date: 2006-08-22 00:00:00 +03:00
 summary: A MediaWiki syntax parser and HTML generator.
 require_paths: 
 - lib
@@ -28,29 +28,20 @@ cert_chain:
 authors: 
 - Pluron Inc.
 files: 
-- lib/mediacloth.rb~
 - lib/mediacloth
 - lib/mediacloth.rb
 - lib/mediacloth/mediawikihtmlgenerator.rb
 - lib/mediacloth/mediawikiparams.rb
 - lib/mediacloth/mediawikiwalker.rb
 - lib/mediacloth/mediawikiparser.rb
-- lib/mediacloth/mediawikiparser.y~
-- lib/mediacloth/mediawikihtmlgenerator.rb~
 - lib/mediacloth/mediawikiparser.y
-- lib/mediacloth/mediawikiwalker.rb~
 - lib/mediacloth/mediawikilexer.rb
 - lib/mediacloth/mediawikiast.rb
 - test/data
-- test/testhelper.rb~
 - test/parser.rb
-- test/parser.rb~
 - test/testhelper.rb
-- test/htmlgenerator.rb~
-- test/debugwalker.rb~
 - test/lexer.rb
 - test/debugwalker.rb
-- test/lexer.rb~
 - test/dataproducers
 - test/htmlgenerator.rb
 - test/data/lex1
@@ -60,13 +51,16 @@ files:
 - test/data/lex5
 - test/data/lex6
 - test/data/lex7
+- test/data/lex8
 - test/data/result1
 - test/data/html1
 - test/data/html2
 - test/data/html3
 - test/data/html4
+- test/data/html5
 - test/data/html6
 - test/data/html7
+- test/data/html8
 - test/data/input1
 - test/data/input2
 - test/data/input3
@@ -74,7 +68,7 @@ files:
 - test/data/input5
 - test/data/input6
 - test/data/input7
-- test/dataproducers/html.rb~
+- test/data/input8
 - test/dataproducers/lex.rb
 - test/dataproducers/html.rb
 - README

data/lib/mediacloth/mediawikihtmlgenerator.rb~

@@ -1,105 +0,0 @@
-require 'mediawikiwalker'
-require 'mediawikiparams'
-
-#HTML generator for a MediaWiki parse tree
-#
-#Typical use case:
-# parser = MediaWikiParser.new
-# parser.lexer = MediaWikiLexer.new
-# ast = parser.parse(input)
-# walker = MediaWikiHTMLGenerator.new
-# walker.parse(ast)
-# puts walker.html
-class MediaWikiHTMLGenerator < MediaWikiWalker
-    attr_reader :html
-
-    def initialize
-        @html = ""
-    end
-
-protected
-
-    def parse_wiki_ast(ast)
-        super(ast)
-    end
-
-    def parse_text(ast)
-        tag = formatting_to_tag(ast)
-        if tag[0].empty?
-            @html += ast.contents
-        else
-            @html += "<#{tag[0]}#{tag[1]}>#{ast.contents}</#{tag[0]}>"
-        end
-        super(ast)
-    end
-
-    def parse_formatted(ast)
-        tag = formatting_to_tag(ast)
-        @html += "<#{tag}>"
-        super(ast)
-        @html += "</#{tag}>"
-    end
-
-    def parse_list(ast)
-        tag = list_tag(ast)
-        @html += "<#{tag}>"
-        super(ast)
-        @html += "</#{tag}>"
-    end
-
-    def parse_list_item(ast)
-        @html += "<li>"
-        super(ast)
-        @html += "</li>"
-    end
-
-    def parse_preformatted(ast)
-        super(ast)
-    end
-
-    def parse_section(ast)
-        @html += "<h#{ast.level}>"
-        @html += ast.contents.strip
-        @html += "</h#{ast.level}>"
-        super(ast)
-    end
-
-private
-
-    #returns an array with a tag name and tag attributes
-    def formatting_to_tag(ast)
-        tag = ["", ""]
-        if ast.formatting == :Bold
-            tag = ["b", ""]
-        elsif ast.formatting == :Italic
-            tag = ["i", ""]
-        elsif ast.formatting == :Link or ast.formatting == :ExternalLink
-            links = ast.contents.split
-            link = links[0]
-            link_name = links[1, links.length-1].join(" ")
-            link_name = link if link_name.empty?
-            ast.contents = link_name
-            tag = ["a", " href=\"#{link}\" rel=\"nofollow\""]
-        elsif ast.formatting == :HLine
-            ast.contents = ""
-            tag = ["hr", ""]
-        elsif ast.formatting == :SignatureDate
-            ast.contents = MediaWikiParams.instance.time.to_s
-        elsif ast.formatting == :SignatureName
-            ast.contents = MediaWikiParams.instance.author
-        elsif ast.formatting == :SignatureFull
-            ast.contents = MediaWikiParams.instance.author + " " + MediaWikiParams.instance.time.to_s
-        end
-        tag
-    end
-
-    #returns a tag name of the list in ast node
-    def list_tag(ast)
-        if ast.type == :Bulleted
-            return "ul"
-        elsif ast.type == :Numbered
-            return "ol"
-        end
-    end
-
-end

data/lib/mediacloth/mediawikiparser.y~

@@ -1,172 +0,0 @@
-#The parser for the MediaWiki language.
-#
-#Usage together with a lexer:
-# inputFile = File.new("data/input1", "r")
-# input = inputFile.read
-# parser = MediaWikiParser.new
-# parser.lexer = MediaWikiLexer.new
-# parser.parse(input)
-class MediaWikiParser
-
-token BOLDSTART BOLDEND ITALICSTART ITALICEND LINKSTART LINKEND
-    INTLINKSTART INTLINKEND SECTION TEXT PRE
-    HLINE SIGNATURE_NAME SIGNATURE_DATE SIGNATURE_FULL
-    UL_START UL_END LI_START LI_END OL_START OL_END
-
-rule
-
-wiki:
-    repeated_contents
-        {
-            @nodes.push WikiAST.new
-            #@nodes.last.children.insert(0, val[0])
-            #puts val[0]
-            @nodes.last.children += val[0]
-        }
-    ;
-
-contents:
-      text
-        {
-            result = val[0]
-        }
-    | bulleted_list
-        {
-            result = val[0]
-        }
-    | preformatted
-        {
-            p = PreformattedAST.new
-            p.contents = val[0]
-            result = p
-        }
-    | section
-        {
-            s = SectionAST.new
-            s.contents = val[0][0]
-            s.level = val[0][1]
-            result = s
-        }
-    ;
-
-repeated_contents: contents
-        {
-            result = []
-            result << val[0]
-        }
-    | repeated_contents contents
-        {
-            result = []
-            result += val[0]
-            result << val[1]
-        }
-    ;
-
-text: element
-        {
-            p = TextAST.new
-            p.formatting = val[0][0]
-            p.contents = val[0][1]
-            result = p
-        }
-    | formatted_element
-        {
-            result = val[0]
-        }
-    ;
-
-element: LINKSTART TEXT LINKEND
-        { return [:Link, val[1]] }
-    | INTLINKSTART TEXT INTLINKEND
-        { return [:InternalLink, val[1]] }
-    | TEXT
-        { return [:None, val[0]] }
-    | HLINE
-        { return [:HLine, val[0]] }
-    | SIGNATURE_DATE
-        { return [:SignatureDate, val[0]] }
-    | SIGNATURE_NAME
-        { return [:SignatureName, val[0]] }
-    | SIGNATURE_FULL
-        { return [:SignatureFull, val[0]] }
-    ;
-
-formatted_element: BOLDSTART repeated_contents BOLDEND
-        {
-            p = FormattedAST.new
-            p.formatting = :Bold
-            p.children += val[1]
-            result = p
-        }
-    | ITALICSTART repeated_contents ITALICEND
-        {
-            p = FormattedAST.new
-            p.formatting = :Italic
-            p.children += val[1]
-            result = p
-        }
-    ;
-
-bulleted_list: UL_START list_item list_contents UL_END
-        {
-            list = ListAST.new
-            list.type = :Bulleted
-            list.children << val[1]
-            list.children += val[2]
-            result = list
-        }
-    ;
-
-list_contents:
-        { result = [] }
-    list_item list_contents
-        {
-            result << val[1]
-            result += val[2]
-        }
-    |
-        { result = [] }
-    ;
-
-list_item: LI_START repeated_contents LI_END
-        {
-            li = ListItemAST.new
-            li.children += val[1]
-            result = li
-        }
-    ;
-
-preformatted: PRE
-        { result = val[0] }
-    ;
-
-section: SECTION TEXT SECTION
-        { result = [val[1], val[0].length] }
-    ;
-
-end
-
----- header ----
-require 'mediawikiast'
-
----- inner ----
-
-attr_accessor :lexer
-
-def initialize
-    @nodes = []
-    super
-end
-
-#Tokenizes input string and parses it.
-def parse(input)
-    @yydebug=true
-    lexer.tokenize(input)
-    do_parse
-    return @nodes.last
-end
-
-#Asks the lexer to return the next token.
-def next_token
-    return @lexer.lex
-end

data/lib/mediacloth/mediawikiwalker.rb~

@@ -1,62 +0,0 @@
-require 'mediawikiast'
-
-#Default walker to traverse the parse tree.
-#
-#The walker traverses the entire parse tree and does nothing.
-#To implement some functionality during this process, reimplement
-#<i>parse...</i> methods and don't forget to call super() to not
-#break the walk.
-#
-#Current implementations: MediaWikiHTMLGenerator, DebugWalker
-class MediaWikiWalker
-
-    #Walks through the AST
-    def parse(ast)
-        parse_wiki_ast(ast)
-    end
-
-protected
-
-#===== reimplement these methods and don't forget to call super() ====#
-
-    #Reimplement this
-    def parse_wiki_ast(ast)
-        ast.children.each do |c|
-            parse_formatted(c) if c.class == FormattedAST
-            parse_text(c) if c.class == TextAST
-            parse_list(c) if c.class == ListAST
-            parse_preformatted(c) if c.class == PreformattedAST
-            parse_section(c) if c.class == SectionAST
-        end
-    end
-
-    #Reimplement this
-    def parse_formatted(ast)
-        parse_wiki_ast(ast)
-    end
-
-    #Reimplement this
-    def parse_text(ast)
-    end
-
-    #Reimplement this
-    def parse_list(ast)
-        ast.children.each do |c|
-            parse_list_item(c) if c.class == ListItemAST
-        end
-    end
-
-    #Reimplement this
-    def parse_list_item(ast)
-        parse_wiki_ast(ast)
-    end
-
-    #Reimplement this
-    def parse_preformatted(ast)
-    end
-
-    #Reimplement this
-    def parse_section(ast)
-    end
-
-end