mediacloth 0.0.2 → 0.0.3

data/test/data/html9

@@ -0,0 +1,6 @@
+<p>Some examples of wiki escaping.</p><p>A paragraph with some wiki characters: ''italic'' '''bold''' [[Link]]</p><p>An escaped list:
+
+* one
+* two
+* three
+</p><p>Text escaped in a heading:</p><h2>Heading with '''</h2>

data/test/data/input1

@@ -19,6 +19,11 @@ We can have headlines:
 ====== Headline6 ======
 ======= Headline7 =======
 
+Headlines may have formatting:
+
+= See [[Internal Link]] for more info =
+== This is an '''important''' heading ==
+
  This is a preformatted ''' ''' << '' '' text
  yes
 

data/test/data/input10

@@ -0,0 +1,124 @@
+Some examples of tables.
+
+{|
+|}
+
+{|
+|-
+|}
+
+{|
+| a
+| b
+|}
+
+{|
+| a
+| b
+|-
+| 1
+| 2
+|}
+
+{|
+| a || b
+|-
+| 1 || 2
+|}
+
+{|
+! a
+! b
+|-
+| 1
+| 2
+|}
+
+{|
+! a || b
+|-
+| 1 || 2
+|}
+
+{|
+| a
+|-
+| 1
+| 2
+|}
+
+{|
+| a
+| b
+|-
+| 1
+|}
+
+{|
+| a
+| b
+|-
+|
+| 2
+|}
+
+{|
+| [http://example.com Example]
+| '''bold'''
+|-
+| [[example|Example]]
+| [[image:example.jpg|1|2|3]]
+|}
+
+{|
+| [http://example.com Example
+| '''bold
+|-
+| [[example|Example
+| '''bold ''italic
+|}
+
+{|
+| a
+| b
+|-
+| A list in a cell:
+* one
+* two
+|}
+
+{|
+| a
+| b
+|-
+| A table in a cell:
+{|
+| 1
+| 2
+|-
+| one
+| two
+|}
+|}
+
+
+{| border="1"
+! a
+! b
+|- align="left"
+| 1
+| 2
+|}
+
+
+Text before
+{|
+| a
+| b
+|}
+
+Text before with ''italic
+{|
+| a
+| b
+|}

data/test/data/input4

@@ -1 +1,50 @@
-'''''Foo'''''
+Test mixing of bold and italic formatting:
+
+''italic'''bold'''italic''
+
+'''bold''italic''bold'''
+
+'''''boldalic'''''
+
+''italic'''bold'''italic'''''bold''italic''bold'''
+
+
+Test bold and italic wrapped around inline links:
+
+''http://example.com''
+
+'''http://example.com'''
+
+''http://example.com'''
+
+'''http://example.com' is good'''
+
+''http://example.com' is good''
+
+
+Test unclosed bold and italic formatting:
+
+''Some italic and now ''bold
+==Heading==
+Text
+
+
+''Some italic and now '''bold
+==Heading==
+Text
+
+
+''Some italic and now '''bold
+
+==Heading==
+Text
+
+
+''Some italic and now '''bold
+* one
+* two
+
+
+''Some italic and now '''bold
+
+Text

data/test/data/input5

@@ -12,3 +12,11 @@
 ####a
 #a
 ##aa
+
+*A
+*B
+*
+
+#A
+#B
+#

data/test/data/input7

@@ -1,2 +1,35 @@
-[http://www.example.com external link]
-http://www.example.com [foo]
+http://sun.com
+
+[http://sun.com]
+
+[mailto:joe@sun.com]
+
+[http://sun.com stars]
+
+[http://sun.com stars and moon]
+
+[http://sun.com stars and ''moon'' and '''trees''' and birds]
+
+[[sun]]
+
+[[sun|All about Sun]]
+
+[[image:sun|All about Sun]]
+
+[[sun|nofollow|All about Sun]]
+
+[[image:sun|nofollow|All about Sun]]
+
+[[image:sun|All about [[Sun]]]]
+
+[[image:sun|All about [[sun|More about]]]]
+
+[[image:sun|one||two||three|]]
+
+[]
+
+[ ]
+
+[[]]
+
+[[ ]]

data/test/data/input9

@@ -0,0 +1,14 @@
+Some examples of wiki escaping.
+
+A paragraph with some wiki characters: <nowiki>''italic'' '''bold''' [[Link]]</nowiki>
+
+An escaped list:
+<nowiki>
+* one
+* two
+* three
+</nowiki>
+
+Text escaped in a heading:
+
+==Heading with <nowiki>'''</nowiki>==

data/test/data/lex1

@@ -6,12 +6,16 @@ PARA_STARTTEXTAnother paragraph starts after a blank line.PARA_END
 PARA_STARTTEXT
 Another one.PARA_END
 
-PARA_STARTTEXTThis is text with INTLINKSTART[[TEXTInternal LinkINTLINKEND]]TEXT and LINKSTART[TEXThttp://www.example.com external linkLINKEND]TEXT.PARA_END
+PARA_STARTTEXTThis is text with INTLINKSTART[[TEXTInternal LinkINTLINKEND]]TEXT and LINKSTART[TEXThttp://www.example.comLINKSEPTEXTexternal linkLINKEND]TEXT.PARA_END
 
 PARA_STARTTEXTWe can have headlines:PARA_END
 
 SECTION_START=TEXT Headline1 SECTION_END=SECTION_START==TEXT Headline2 SECTION_END==SECTION_START===TEXT Headline3 SECTION_END===SECTION_START====TEXT Headline4 SECTION_END====SECTION_START=====TEXT Headline5 SECTION_END=====SECTION_START======TEXT Headline6 SECTION_END======SECTION_START=======TEXT Headline7 SECTION_END=======PARA_STARTPARA_END
 
+PARA_STARTTEXTHeadlines may have formatting:PARA_END
+
+SECTION_START=TEXT See INTLINKSTART[[TEXTInternal LinkINTLINKEND]]TEXT for more info SECTION_END=SECTION_START==TEXT This is an BOLDSTART'''TEXTimportantBOLDEND'''TEXT heading SECTION_END==PARA_STARTPARA_END
+
 PREThis is a preformatted ''' ''' << '' '' text
 PREyes
 HLINE----PARA_STARTTEXT

data/test/data/lex10

@@ -0,0 +1,87 @@
+PARA_STARTTEXTSome examples of tables.PARA_END
+
+TABLE_START{|TABLE_END|}PARA_STARTPARA_END
+
+TABLE_START{|ROW_START|-ROW_END|}TABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START|CELL_STARTTEXT a
+CELL_END|CELL_STARTTEXT b
+CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START|CELL_STARTTEXT a
+CELL_END|CELL_STARTTEXT b
+CELL_END|-ROW_ENDROW_STARTCELL_START|TEXT 1
+CELL_END|CELL_STARTTEXT 2
+CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START|CELL_STARTTEXT a CELL_END||CELL_STARTTEXT b
+CELL_END|-ROW_ENDROW_STARTCELL_START|TEXT 1 CELL_END||CELL_STARTTEXT 2
+CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START!HEAD_STARTTEXT a
+HEAD_END!HEAD_STARTTEXT b
+HEAD_END|-ROW_ENDROW_STARTCELL_START|TEXT 1
+CELL_END|CELL_STARTTEXT 2
+CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START!HEAD_STARTTEXT a HEAD_END||HEAD_STARTTEXT b
+HEAD_END|-ROW_ENDROW_STARTCELL_START|TEXT 1 CELL_END||CELL_STARTTEXT 2
+CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START|CELL_STARTTEXT a
+CELL_END|-ROW_ENDROW_STARTCELL_START|TEXT 1
+CELL_END|CELL_STARTTEXT 2
+CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START|CELL_STARTTEXT a
+CELL_END|CELL_STARTTEXT b
+CELL_END|-ROW_ENDROW_STARTCELL_START|TEXT 1
+CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START|CELL_STARTTEXT a
+CELL_END|CELL_STARTTEXT b
+CELL_END|-ROW_ENDROW_STARTCELL_START|CELL_END|CELL_STARTTEXT 2
+CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START|CELL_STARTTEXT LINKSTART[TEXThttp://example.comLINKSEPTEXTExampleLINKEND]CELL_END|CELL_STARTTEXT BOLDSTART'''TEXTboldBOLDEND'''CELL_END|-ROW_ENDROW_STARTCELL_START|TEXT INTLINKSTART[[TEXTexampleINTLINKSEP|TEXTExampleINTLINKEND]]CELL_END|CELL_STARTTEXT INTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTexample.jpgINTLINKSEP|TEXT1INTLINKSEP|TEXT2INTLINKSEP|TEXT3INTLINKEND]]CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START|CELL_STARTTEXT LINKSTART[TEXThttp://example.comLINKSEPTEXTExample
+LINKEND|CELL_ENDCELL_STARTTEXT BOLDSTART'''TEXTbold
+BOLDEND|-CELL_ENDROW_ENDROW_STARTCELL_START|TEXT INTLINKSTART[[TEXTexampleINTLINKSEP|TEXTExample
+INTLINKEND|CELL_ENDCELL_STARTTEXT BOLDSTART'''TEXTbold ITALICSTART''TEXTitalic
+ITALICEND|}BOLDENDCELL_ENDROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START|CELL_STARTTEXT a
+CELL_END|CELL_STARTTEXT b
+CELL_END|-ROW_ENDROW_STARTCELL_START|TEXT A list in a cell:
+UL_STARTLI_STARTTEXTone
+LI_ENDLI_STARTTEXTtwo
+LI_ENDUL_ENDCELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|ROW_START|CELL_STARTTEXT a
+CELL_END|CELL_STARTTEXT b
+CELL_END|-ROW_ENDROW_STARTCELL_START|TEXT A table in a cell:
+TABLE_START{|ROW_START|CELL_STARTTEXT 1
+CELL_END|CELL_STARTTEXT 2
+CELL_END|-ROW_ENDROW_STARTCELL_START|TEXT one
+CELL_END|CELL_STARTTEXT two
+CELL_END|}ROW_ENDTABLE_ENDCELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+TABLE_START{|TEXT border="1"
+ROW_START!HEAD_STARTTEXT a
+HEAD_END!HEAD_STARTTEXT b
+HEAD_END|-ROW_ENDROW_STARTTEXT align="left"
+CELL_START|TEXT 1
+CELL_END|CELL_STARTTEXT 2
+CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+PARA_STARTTEXT
+Text before
+PARA_END{|TABLE_STARTROW_START|CELL_STARTTEXT a
+CELL_END|CELL_STARTTEXT b
+CELL_END|}ROW_ENDTABLE_ENDPARA_STARTPARA_END
+
+PARA_STARTTEXTText before with ITALICSTART''TEXTitalic
+ITALICEND{|PARA_ENDTABLE_STARTROW_START|CELL_STARTTEXT a
+CELL_END|CELL_STARTTEXT b
+CELL_END|}ROW_ENDTABLE_ENDfalsefalse

data/test/data/lex4

@@ -1 +1,47 @@
-PARA_STARTBOLDSTART'''ITALICSTART''TEXTFooITALICEND''BOLDEND'''PARA_ENDfalsefalse
+PARA_STARTTEXTTest mixing of bold and italic formatting:PARA_END
+
+PARA_STARTITALICSTART''TEXTitalicBOLDSTART'''TEXTboldBOLDEND'''TEXTitalicITALICEND''PARA_END
+
+PARA_STARTBOLDSTART'''TEXTboldITALICSTART''TEXTitalicITALICEND''TEXTboldBOLDEND'''PARA_END
+
+PARA_STARTITALICSTART''BOLDSTART'''TEXTboldalicBOLDEND'''ITALICEND''PARA_END
+
+PARA_STARTITALICSTART''TEXTitalicBOLDSTART'''TEXTboldBOLDEND'''TEXTitalicITALICEND''BOLDSTART'''TEXTboldITALICSTART''TEXTitalicITALICEND''TEXTboldBOLDEND'''PARA_END
+
+PARA_STARTTEXT
+Test bold and italic wrapped around inline links:PARA_END
+
+PARA_STARTITALICSTART''LINKSTARTTEXThttp://example.comLINKEND]ITALICEND''PARA_END
+
+PARA_STARTBOLDSTART'''LINKSTARTTEXThttp://example.comLINKEND]BOLDEND'''PARA_END
+
+PARA_STARTITALICSTART''LINKSTARTTEXThttp://example.com'LINKEND]ITALICEND''PARA_END
+
+PARA_STARTBOLDSTART'''LINKSTARTTEXThttp://example.com'LINKEND]TEXT is goodBOLDEND'''PARA_END
+
+PARA_STARTITALICSTART''LINKSTARTTEXThttp://example.com'LINKEND]TEXT is goodITALICEND''PARA_END
+
+PARA_STARTTEXT
+Test unclosed bold and italic formatting:PARA_END
+
+PARA_STARTITALICSTART''TEXTSome italic and now ITALICEND''TEXTbold
+PARA_ENDSECTION_START==TEXTHeadingSECTION_END==PARA_STARTTEXT
+TextPARA_END
+
+PARA_STARTITALICSTART''TEXTSome italic and now BOLDSTART'''TEXTbold
+BOLDENDITALICENDPARA_ENDSECTION_START==TEXTHeadingSECTION_END==PARA_STARTTEXT
+TextPARA_END
+
+PARA_STARTITALICSTART''TEXTSome italic and now BOLDSTART'''TEXTboldBOLDEND
+
+ITALICENDPARA_ENDSECTION_START==TEXTHeadingSECTION_END==PARA_STARTTEXT
+TextPARA_END
+
+PARA_STARTITALICSTART''TEXTSome italic and now BOLDSTART'''TEXTbold
+BOLDENDITALICENDPARA_ENDUL_STARTLI_STARTTEXTone
+LI_ENDLI_STARTTEXTtwo
+LI_ENDUL_ENDPARA_STARTPARA_END
+
+PARA_STARTITALICSTART''TEXTSome italic and now BOLDSTART'''TEXTboldBOLDEND
+
+ITALICENDPARA_ENDPARA_STARTTEXTTextPARA_ENDfalsefalse

data/test/data/lex5

@@ -9,4 +9,10 @@ LI_ENDUL_ENDLI_ENDUL_ENDOL_STARTLI_STARTTEXTa
 OL_STARTLI_STARTTEXTa
 OL_STARTLI_STARTTEXTa
 OL_STARTLI_STARTTEXTa
-LI_ENDOL_ENDLI_ENDOL_ENDLI_ENDOL_ENDLI_ENDOL_ENDfalsefalse
+LI_ENDOL_ENDLI_ENDOL_ENDLI_ENDOL_ENDLI_ENDLI_STARTTEXTa
+OL_STARTLI_STARTTEXTaa
+LI_ENDOL_ENDLI_ENDOL_ENDUL_STARTLI_STARTTEXTA
+LI_ENDLI_STARTTEXTB
+LI_ENDLI_STARTLI_ENDUL_ENDOL_STARTLI_STARTTEXTA
+LI_ENDLI_STARTTEXTB
+LI_ENDLI_STARTLI_ENDOL_ENDfalsefalse

data/test/data/lex7

@@ -1,2 +1,35 @@
-PARA_STARTLINKSTART[TEXThttp://www.example.com external linkLINKEND]LINKSTARTTEXThttp://www.example.comLINKEND]TEXT [foo]
-PARA_ENDfalsefalse
+PARA_STARTLINKSTARTTEXThttp://sun.comLINKEND]PARA_END
+
+PARA_STARTLINKSTART[TEXThttp://sun.comLINKEND]PARA_END
+
+PARA_STARTLINKSTART[TEXTmailto:joe@sun.comLINKEND]PARA_END
+
+PARA_STARTLINKSTART[TEXThttp://sun.comLINKSEPTEXTstarsLINKEND]PARA_END
+
+PARA_STARTLINKSTART[TEXThttp://sun.comLINKSEPTEXTstars and moonLINKEND]PARA_END
+
+PARA_STARTLINKSTART[TEXThttp://sun.comLINKSEPTEXTstars and ITALICSTART''TEXTmoonITALICEND''LINKSEPTEXTand BOLDSTART'''TEXTtreesBOLDEND'''LINKSEPTEXTand birdsLINKEND]PARA_END
+
+PARA_STARTINTLINKSTART[[TEXTsunINTLINKEND]]PARA_END
+
+PARA_STARTINTLINKSTART[[TEXTsunINTLINKSEP|TEXTAll about SunINTLINKEND]]PARA_END
+
+PARA_STARTINTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTsunINTLINKSEP|TEXTAll about SunINTLINKEND]]PARA_END
+
+PARA_STARTINTLINKSTART[[TEXTsunINTLINKSEP|TEXTnofollow|All about SunINTLINKEND]]PARA_END
+
+PARA_STARTINTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTsunINTLINKSEP|TEXTnofollowINTLINKSEP|TEXTAll about SunINTLINKEND]]PARA_END
+
+PARA_STARTINTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTsunINTLINKSEP|TEXTAll about INTLINKSTART[[TEXTSunINTLINKEND]]INTLINKEND]]PARA_END
+
+PARA_STARTINTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTsunINTLINKSEP|TEXTAll about INTLINKSTART[[TEXTsunINTLINKSEP|TEXTMore aboutINTLINKEND]]INTLINKEND]]PARA_END
+
+PARA_STARTINTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTsunINTLINKSEP|TEXToneINTLINKSEP|INTLINKSEP|TEXTtwoINTLINKSEP|INTLINKSEP|TEXTthreeINTLINKSEP|INTLINKEND]]PARA_END
+
+PARA_STARTTEXT[]PARA_END
+
+PARA_STARTTEXT[ ]PARA_END
+
+PARA_STARTTEXT[[]]PARA_END
+
+PARA_STARTTEXT[[ ]]PARA_ENDfalsefalse

data/test/data/lex9

@@ -0,0 +1,14 @@
+PARA_STARTTEXTSome examples of wiki escaping.PARA_END
+
+PARA_STARTTEXTA paragraph with some wiki characters: ''italic'' '''bold''' [[Link]]PARA_END
+
+PARA_STARTTEXTAn escaped list:
+
+* one
+* two
+* three
+PARA_END
+
+PARA_STARTTEXTText escaped in a heading:PARA_END
+
+SECTION_START==TEXTHeading with '''SECTION_END==falsefalse

data/test/dataproducers/html.rb

@@ -4,8 +4,8 @@ require 'mediacloth/mediawikiparams'
 require 'mediacloth/mediawikihtmlgenerator'
 
 def produce(index)
-    file = File.new("../data/html#{index}", "w")
-    inputFile = File.new("../data/input#{index}", "r")
+    file = File.new("../test/data/html#{index}", "w")
+    inputFile = File.new("../test/data/input#{index}", "r")
     input = inputFile.read
 
     parser = MediaWikiParser.new

data/test/dataproducers/html.rb~

@@ -0,0 +1,24 @@
+require 'mediacloth/mediawikilexer'
+require 'mediacloth/mediawikiparser'
+require 'mediacloth/mediawikiparams'
+require 'mediacloth/mediawikihtmlgenerator'
+
+def produce(index)
+    file = File.new("../data/html#{index}", "w")
+    inputFile = File.new("../data/input#{index}", "r")
+    input = inputFile.read
+
+    parser = MediaWikiParser.new
+    parser.lexer = MediaWikiLexer.new
+    ast = parser.parse(input)
+    MediaWikiParams.instance.time = Time.mktime(2000, 1, 1, 1, 1, 1, 1)
+    generator = MediaWikiHTMLGenerator.new
+    generator.parse(ast)
+
+#    puts generator.html
+    file.write(generator.html)
+    file.close
+end
+
+# (3..5).each { |i| produce(i) }
+produce(5)

data/test/dataproducers/lex.rb

@@ -1,8 +1,8 @@
 require 'mediacloth/mediawikilexer'
 
 def produce(index)
-    file = File.new("../data/lex#{index}", "w")
-    inputFile = File.new("../data/input#{index}", "r")
+    file = File.new("../test/data/lex#{index}", "w")
+    inputFile = File.new("../test/data/input#{index}", "r")
     input = inputFile.read
 
     lexer = MediaWikiLexer.new
@@ -12,4 +12,4 @@ def produce(index)
 end
 
 #1..5.each { |i| produce(1) }
-produce(8)
+produce(5)

data/test/dataproducers/lex.rb~

@@ -0,0 +1,15 @@
+require 'mediacloth/mediawikilexer'
+
+def produce(index)
+    file = File.new("../data/lex#{index}", "w")
+    inputFile = File.new("../data/input#{index}", "r")
+    input = inputFile.read
+
+    lexer = MediaWikiLexer.new
+    tokens = lexer.tokenize(input)
+    file.write(tokens.to_s)
+    file.close
+end
+
+#1..5.each { |i| produce(1) }
+produce(8)

data/test/debugwalker.rb

@@ -37,7 +37,7 @@ protected
     end
 
     def parse_list(ast)
-        info(ast, ast.type)
+        info(ast, ast.class)
         super(ast)
     end
 

data/test/htmlgenerator.rb

@@ -5,19 +5,20 @@ require 'mediacloth/mediawikihtmlgenerator'
 require 'test/unit'
 require 'testhelper'
 
-class Parser_Test < Test::Unit::TestCase
+class HTMLGenerator_Test < Test::Unit::TestCase
 
     include TestHelper
 
     def test_input
-        test_files("html") { |input,result|
+        test_files("html") { |input,result,name|
+            puts name
             parser = MediaWikiParser.new
             parser.lexer = MediaWikiLexer.new
             ast = parser.parse(input)
-            MediaWikiParams.instance.time = Time.mktime(2000, 1, 1, 1, 1, 1, 1)
+            MediaWikiParams.instance.time = Time.utc(2000, 1, 1, 1, 1, 1, 1)
             generator = MediaWikiHTMLGenerator.new
             generator.parse(ast)
-            assert_equal generator.html, result
+            assert_equal generator.html, result, name
 #             puts generator.html
         }
     end

data/test/lexer.rb

@@ -6,18 +6,35 @@ class Lexer_Test < Test::Unit::TestCase
 
     include TestHelper
 
-    def test_input
+    def test_standard_formatted_input
         test_files("lex") { |input,result,resultname|
             lexer = MediaWikiLexer.new
             tokens = lexer.tokenize(input)
-            assert_equal(tokens.to_s, result)
+            assert_equal(result, tokens.to_s, "Mismatch in #{resultname}")
+        }
+    end
+    
+    def test_internet_formatted_input
+        test_files("lex") { |input,result,resultname|
+            lexer = MediaWikiLexer.new
+            tokens = lexer.tokenize(input.gsub("\n", "\r\n"))
+            assert_equal(result.gsub("\n", "\r\n"), tokens.to_s, "Mismatch in #{resultname}")
         }
     end
 
     def test_paragraphs
-        assert_equal(lex("Before\n\n\n=Headline="),
+        assert_equal(lex("text\n\ntext"),
+            [[:PARA_START, ""], [:TEXT, "text"], [:PARA_END, "\n\n"], 
+              [:PARA_START, ""], [:TEXT, "text"], [:PARA_END, ""], [false,false]])
+        assert_equal(lex("text\r\n\r\ntext"),
+            [[:PARA_START, ""], [:TEXT, "text"], [:PARA_END, "\r\n\r\n"], 
+              [:PARA_START, ""], [:TEXT, "text"], [:PARA_END, ""], [false,false]])
+        assert_equal(lex("Before\n\n=Headline="),
             [[:PARA_START, ""], [:TEXT, "Before"], [:PARA_END, "\n\n"],
              [:SECTION_START, "="], [:TEXT, "Headline"], [:SECTION_END, "="], [false,false]])
+        assert_equal(lex("Before\r\n\r\n=Headline="),
+            [[:PARA_START, ""], [:TEXT, "Before"], [:PARA_END, "\r\n\r\n"],
+             [:SECTION_START, "="], [:TEXT, "Headline"], [:SECTION_END, "="], [false,false]])
     end
 
     def test_empty
@@ -27,6 +44,7 @@ class Lexer_Test < Test::Unit::TestCase
     def test_preformatted
         #assure preformatted text works as expected at the start of the text
         assert_equal(lex(" Foo\n"), [[:PRE, "Foo\n"], [false, false]])
+        assert_equal(lex(" Foo\r\n"), [[:PRE, "Foo\r\n"], [false, false]])
         assert_equal(lex(" Foo"), [[:PRE, "Foo"], [false, false]])
     end
 
@@ -34,16 +52,30 @@ class Lexer_Test < Test::Unit::TestCase
         #assure that at the start of the text hline still works
         assert_equal(lex("----"), [[:HLINE, "----"], [false, false]])
         assert_equal(lex("\n----"), [[:HLINE, "----"], [false, false]])
+        assert_equal(lex("\r\n----"), [[:HLINE, "----"], [false, false]])
     end
+    
+    def test_inline_links
+        #assure that links in-line work 
+        assert_equal(lex("http://example.com"), [[:PARA_START, ""], [:LINKSTART, ""], [:TEXT, "http://example.com"], [:LINKEND, "]"], [:PARA_END, ""], [false, false]])
+        assert_equal(lex("http://example.com\n"), [[:PARA_START, ""], [:LINKSTART, ""], [:TEXT, "http://example.com"], [:LINKEND, "]"], [:PARA_END, ""], [false, false]])
+        #assert_equal(lex("http://example.com''italic''"), [[:PARA_START, ""], [:LINKSTART, ""], [:TEXT, "http://example.com"], [:LINKEND, "]"], [:PARA_END, ""], [false, false]])
+     end
 
     def test_ending_text_token
         #check for a problem when the last token is TEXT and it's not included
         assert_equal(lex("\n----\nfoo\n"),
             [[:HLINE, "----"], [:PARA_START, ""],
                 [:TEXT, "\nfoo\n"], [:PARA_END, ""], [false, false]])
+        assert_equal(lex("\r\n----\r\nfoo\r\n"),
+            [[:HLINE, "----"], [:PARA_START, ""],
+                [:TEXT, "\r\nfoo\r\n"], [:PARA_END, ""], [false, false]])
         assert_equal(lex("\n----\nfoo\n Hehe"),
             [[:HLINE, "----"], [:PARA_START, ""], [:TEXT, "\nfoo\n"],
                 [:PARA_END, ""], [:PRE, "Hehe"], [false, false]])
+        assert_equal(lex("\r\n----\r\nfoo\r\n Hehe"),
+            [[:HLINE, "----"], [:PARA_START, ""], [:TEXT, "\r\nfoo\r\n"],
+                [:PARA_END, ""], [:PRE, "Hehe"], [false, false]])
     end
 
     def test_bullets
@@ -57,6 +89,11 @@ class Lexer_Test < Test::Unit::TestCase
             [:UL_END, ""], [:LI_END, ""], [:UL_END, ""], [false, false]])
     end
 
+    def test_bullets_at_eof
+        assert_equal(lex("* Foo\n*"),
+            [[:UL_START, nil], [:LI_START, ""], [:TEXT, "Foo\n"], [:LI_END, ""], [:LI_START, ""], [:LI_END, ""], [:UL_END, ""], [false, false]])
+    end
+
 private
     def lex(string)
         lexer = MediaWikiLexer.new