mediacloth 0.0.3 → 0.5

data/test/data/lex4

@@ -1,47 +1,53 @@
-PARA_STARTTEXTTest mixing of bold and italic formatting:PARA_END
+PARA_START[0, 0]TEXTTest mixing of bold and italic formatting:
 
-PARA_STARTITALICSTART''TEXTitalicBOLDSTART'''TEXTboldBOLDEND'''TEXTitalicITALICEND''PARA_END
+[0, 44]PARA_END[44, 0]PARA_START[44, 0]ITALIC_START''[44, 2]TEXTitalic[46, 6]BOLD_START'''[52, 3]TEXTbold[55, 4]BOLD_END'''[59, 3]TEXTitalic[62, 6]ITALIC_END''[68, 2]TEXT
 
-PARA_STARTBOLDSTART'''TEXTboldITALICSTART''TEXTitalicITALICEND''TEXTboldBOLDEND'''PARA_END
+[70, 2]PARA_END[72, 0]PARA_START[72, 0]BOLD_START'''[72, 3]TEXTbold[75, 4]ITALIC_START''[79, 2]TEXTitalic[81, 6]ITALIC_END''[87, 2]TEXTbold[89, 4]BOLD_END'''[93, 3]TEXT
 
-PARA_STARTITALICSTART''BOLDSTART'''TEXTboldalicBOLDEND'''ITALICEND''PARA_END
+[96, 2]PARA_END[98, 0]PARA_START[98, 0]ITALIC_START''[98, 2]BOLD_START'''[100, 3]TEXTboldalic[103, 8]BOLD_END'''[111, 3]ITALIC_END''[114, 2]TEXT
 
-PARA_STARTITALICSTART''TEXTitalicBOLDSTART'''TEXTboldBOLDEND'''TEXTitalicITALICEND''BOLDSTART'''TEXTboldITALICSTART''TEXTitalicITALICEND''TEXTboldBOLDEND'''PARA_END
+[116, 2]PARA_END[118, 0]PARA_START[118, 0]ITALIC_START''[118, 2]TEXTitalic[120, 6]BOLD_START'''[126, 3]TEXTbold[129, 4]BOLD_END'''[133, 3]TEXTitalic[136, 6]ITALIC_END''[142, 2]BOLD_START'''[144, 3]TEXTbold[147, 4]ITALIC_START''[151, 2]TEXTitalic[153, 6]ITALIC_END''[159, 2]TEXTbold[161, 4]BOLD_END'''[165, 3]TEXT
 
-PARA_STARTTEXT
-Test bold and italic wrapped around inline links:PARA_END
+[168, 2]PARA_END[170, 0]PARA_START[170, 0]TEXT
+Test bold and italic wrapped around inline links:
 
-PARA_STARTITALICSTART''LINKSTARTTEXThttp://example.comLINKEND]ITALICEND''PARA_END
+[170, 52]PARA_END[222, 0]PARA_START[222, 0]ITALIC_START''[222, 2]LINK_START[224, 0]TEXThttp://example.com[224, 18]LINK_END[242, 0]ITALIC_END''[242, 2]TEXT
 
-PARA_STARTBOLDSTART'''LINKSTARTTEXThttp://example.comLINKEND]BOLDEND'''PARA_END
+[244, 2]PARA_END[246, 0]PARA_START[246, 0]BOLD_START'''[246, 3]LINK_START[249, 0]TEXThttp://example.com[249, 18]LINK_END[267, 0]BOLD_END'''[267, 3]TEXT
 
-PARA_STARTITALICSTART''LINKSTARTTEXThttp://example.com'LINKEND]ITALICEND''PARA_END
+[270, 2]PARA_END[272, 0]PARA_START[272, 0]ITALIC_START''[272, 2]LINK_START[274, 0]TEXThttp://example.com[274, 18]LINK_END[292, 0]BOLD_START'''[292, 3]TEXT
 
-PARA_STARTBOLDSTART'''LINKSTARTTEXThttp://example.com'LINKEND]TEXT is goodBOLDEND'''PARA_END
+[295, 2]BOLD_END[297, 0]ITALIC_END[297, 0]PARA_END[297, 0]PARA_START[297, 0]BOLD_START'''[297, 3]LINK_START[300, 0]TEXThttp://example.com[300, 18]LINK_END[318, 0]TEXT' is good[318, 9]BOLD_END'''[327, 3]TEXT
 
-PARA_STARTITALICSTART''LINKSTARTTEXThttp://example.com'LINKEND]TEXT is goodITALICEND''PARA_END
+[330, 2]PARA_END[332, 0]PARA_START[332, 0]ITALIC_START''[332, 2]LINK_START[334, 0]TEXThttp://example.com[334, 18]LINK_END[352, 0]TEXT' is good[352, 9]ITALIC_END''[361, 2]TEXT
 
-PARA_STARTTEXT
-Test unclosed bold and italic formatting:PARA_END
+[363, 2]PARA_END[365, 0]PARA_START[365, 0]TEXT
+Test unclosed bold and italic formatting:
 
-PARA_STARTITALICSTART''TEXTSome italic and now ITALICEND''TEXTbold
-PARA_ENDSECTION_START==TEXTHeadingSECTION_END==PARA_STARTTEXT
-TextPARA_END
+[365, 44]PARA_END[409, 0]PARA_START[409, 0]ITALIC_START''[409, 2]TEXTSome italic and now [411, 20]ITALIC_END''[431, 2]TEXTbold
+[433, 5]PARA_END[438, 0]SECTION_START==[438, 2]TEXTHeading[440, 7]SECTION_END==[447, 0]PARA_START[447, 3]TEXTText
 
-PARA_STARTITALICSTART''TEXTSome italic and now BOLDSTART'''TEXTbold
-BOLDENDITALICENDPARA_ENDSECTION_START==TEXTHeadingSECTION_END==PARA_STARTTEXT
-TextPARA_END
+[450, 6]PARA_END[456, 0]PARA_START[456, 1]ITALIC_START''[457, 2]TEXTSome italic and now [459, 20]BOLD_START'''[479, 3]TEXTbold
+[482, 5]BOLD_END[487, 0]ITALIC_END[487, 0]PARA_END[487, 0]SECTION_START==[487, 2]TEXTHeading[489, 7]SECTION_END==[496, 0]PARA_START[496, 3]TEXT
+Text
 
-PARA_STARTITALICSTART''TEXTSome italic and now BOLDSTART'''TEXTboldBOLDEND
+[499, 7]PARA_END[506, 0]PARA_START[506, 1]ITALIC_START''[507, 2]TEXTSome italic and now [509, 20]BOLD_START'''[529, 3]TEXTbold
 
-ITALICENDPARA_ENDSECTION_START==TEXTHeadingSECTION_END==PARA_STARTTEXT
-TextPARA_END
+[532, 6]BOLD_END[538, 0]ITALIC_END[538, 0]PARA_END[538, 0]SECTION_START==[538, 2]TEXTHeading[540, 7]SECTION_END==[547, 0]PARA_START[547, 3]TEXT
 
-PARA_STARTITALICSTART''TEXTSome italic and now BOLDSTART'''TEXTbold
-BOLDENDITALICENDPARA_ENDUL_STARTLI_STARTTEXTone
-LI_ENDLI_STARTTEXTtwo
-LI_ENDUL_ENDPARA_STARTPARA_END
+[550, 2]PARA_END[552, 0]PARA_START[552, 0]TEXTText
 
-PARA_STARTITALICSTART''TEXTSome italic and now BOLDSTART'''TEXTboldBOLDEND
+[552, 6]PARA_END[558, 0]PARA_START[558, 1]ITALIC_START''[559, 2]TEXTSome italic and now [561, 20]BOLD_START'''[581, 3]TEXTbold
+[584, 5]BOLD_END[589, 0]ITALIC_END[589, 0]PARA_END[589, 0]UL_START[589, 0]LI_START[589, 1]TEXT one
+[590, 5]LI_END[595, 0]LI_START[595, 1]TEXT two
+[596, 5]LI_END[601, 0]UL_END[601, 0]PARA_START[601, 0]TEXT
 
-ITALICENDPARA_ENDPARA_STARTTEXTTextPARA_ENDfalsefalse
+[601, 2]PARA_END[603, 0]PARA_START[603, 0]ITALIC_START''[603, 2]TEXTSome italic and now [605, 20]BOLD_START'''[625, 3]TEXTbold
+
+[628, 6]BOLD_END[634, 0]ITALIC_END[634, 0]PARA_END[634, 0]PARA_START[634, 0]TEXTText
+
+[634, 6]PARA_END[640, 0]PARA_START[640, 0]TEXTInvalid and weird sections:
+[640, 28]PARA_END[668, 0]SECTION_START======[668, 6]TEXT====[674, 4]SECTION_END======[678, 0]PARA_START[678, 6]TEXT
+=
+==
+[684, 6]PARA_END[690, 0]SECTION_START=[690, 1]TEXT=[691, 1]SECTION_END=[692, 2]SECTION_START=[694, 1]TEXT==[695, 2]SECTION_END=[697, 2]SECTION_START==[699, 2]TEXT=[701, 1]SECTION_END==[702, 3]SECTION_START==[705, 2]TEXT==[707, 2]SECTION_END==[709, 3]falsefalse[712, 0]

data/test/data/lex5

@@ -1,18 +1,57 @@
-UL_STARTLI_STARTTEXTA
-UL_STARTLI_STARTTEXTA
-UL_STARTLI_STARTTEXTa
-LI_ENDLI_STARTTEXTb
-UL_STARTLI_STARTTEXTa
-LI_ENDUL_ENDLI_ENDUL_ENDLI_ENDUL_ENDLI_ENDLI_STARTTEXTB
-UL_STARTLI_STARTTEXTb
-LI_ENDUL_ENDLI_ENDUL_ENDOL_STARTLI_STARTTEXTa
-OL_STARTLI_STARTTEXTa
-OL_STARTLI_STARTTEXTa
-OL_STARTLI_STARTTEXTa
-LI_ENDOL_ENDLI_ENDOL_ENDLI_ENDOL_ENDLI_ENDLI_STARTTEXTa
-OL_STARTLI_STARTTEXTaa
-LI_ENDOL_ENDLI_ENDOL_ENDUL_STARTLI_STARTTEXTA
-LI_ENDLI_STARTTEXTB
-LI_ENDLI_STARTLI_ENDUL_ENDOL_STARTLI_STARTTEXTA
-LI_ENDLI_STARTTEXTB
-LI_ENDLI_STARTLI_ENDOL_ENDfalsefalse
+PARA_START[0, 0]TEXTUnordered lists
+
+[0, 17]PARA_END[17, 0]UL_START[17, 0]LI_START[17, 1]TEXTA
+[18, 2]UL_START[20, 0]LI_START[20, 2]TEXTA
+[22, 2]UL_START[24, 0]LI_START[24, 3]TEXTa
+[27, 2]LI_END[29, 0]LI_START[29, 3]TEXTb
+[32, 2]UL_START[34, 0]LI_START[34, 4]TEXTa
+[38, 2]LI_END[40, 0]UL_END[40, 0]LI_END[40, 0]UL_END[40, 0]LI_END[40, 0]UL_END[40, 0]LI_END[40, 0]LI_START[40, 1]TEXTB
+[41, 2]UL_START[43, 0]LI_START[43, 2]TEXTb
+[45, 2]LI_END[47, 0]UL_END[47, 0]LI_END[47, 0]UL_END[47, 1]UL_START[48, 0]LI_START[48, 1]TEXTA
+[49, 2]LI_END[51, 0]LI_START[51, 1]TEXTB
+[52, 2]LI_END[54, 0]LI_START[54, 2]LI_END[56, 0]UL_END[56, 0]PARA_START[56, 0]TEXT
+Ordered lists
+
+[56, 16]PARA_END[72, 0]OL_START[72, 0]LI_START[72, 1]TEXTa
+[73, 2]OL_START[75, 0]LI_START[75, 2]TEXTa
+[77, 2]OL_START[79, 0]LI_START[79, 3]TEXTa
+[82, 2]OL_START[84, 0]LI_START[84, 4]TEXTa
+[88, 2]LI_END[90, 0]OL_END[90, 0]LI_END[90, 0]OL_END[90, 0]LI_END[90, 0]OL_END[90, 0]LI_END[90, 0]LI_START[90, 1]TEXTa
+[91, 2]OL_START[93, 0]LI_START[93, 2]TEXTaa
+[95, 3]LI_END[98, 0]OL_END[98, 0]LI_END[98, 0]OL_END[98, 1]OL_START[99, 0]LI_START[99, 1]TEXTA
+[100, 2]LI_END[102, 0]LI_START[102, 1]TEXTB
+[103, 2]LI_END[105, 0]LI_START[105, 2]LI_END[107, 0]OL_END[107, 0]PARA_START[107, 0]TEXT
+Mixed lists
+
+[107, 14]PARA_END[121, 0]UL_START[121, 0]LI_START[121, 1]TEXTa
+[122, 2]OL_START[124, 0]LI_START[124, 2]TEXT1
+[126, 2]LI_END[128, 0]LI_START[128, 2]TEXT2
+[130, 2]LI_END[132, 0]OL_END[132, 0]LI_END[132, 0]LI_START[132, 1]TEXTb
+[133, 2]LI_END[135, 0]UL_END[135, 1]OL_START[136, 0]LI_START[136, 1]TEXTa
+[137, 2]UL_START[139, 0]LI_START[139, 2]TEXT1
+[141, 2]LI_END[143, 0]LI_START[143, 2]TEXT2
+[145, 2]UL_START[147, 0]LI_START[147, 3]TEXT2
+[150, 2]LI_END[152, 0]UL_END[152, 0]LI_END[152, 0]UL_END[152, 0]LI_END[152, 0]LI_START[152, 1]TEXTb
+[153, 2]LI_END[155, 0]OL_END[155, 0]PARA_START[155, 0]TEXT
+Dictionary lists
+
+[155, 19]PARA_END[174, 0]DL_START[174, 0]DT_START;[174, 1]TEXTd
+[175, 2]DT_END[177, 0]DL_END[177, 0]PARA_START[177, 0]TEXT
+
+[177, 2]PARA_END[179, 0]DL_START[179, 0]DD_START:[179, 1]TEXTt
+[180, 2]DD_END[182, 0]DL_END[182, 0]PARA_START[182, 0]TEXT
+
+[182, 2]PARA_END[184, 0]DL_START[184, 0]DT_START;[184, 1]TEXTd
+[185, 2]DT_END[187, 0]DD_START:[187, 1]TEXTt
+[188, 2]DD_END[190, 0]DL_END[190, 0]PARA_START[190, 0]TEXT
+
+[190, 2]PARA_END[192, 0]DL_START[192, 0]DT_START;[192, 1]TEXTd
+[193, 2]DT_END[195, 0]DD_START:[195, 1]TEXTt
+[196, 2]DD_END[198, 0]DD_START:[198, 1]TEXTt
+[199, 2]DD_END[201, 0]DL_END[201, 0]PARA_START[201, 0]TEXT
+
+[201, 2]PARA_END[203, 0]DL_START[203, 0]DT_START;[203, 1]TEXTd
+[204, 2]DT_END[206, 0]DD_START:[206, 1]TEXTt
+[207, 2]DD_END[209, 0]DL_END[209, 0]DL_START[209, 0]DT_START;[209, 1]TEXTd
+[210, 2]DT_END[212, 0]DD_START:[212, 1]TEXTt
+[213, 2]DD_END[215, 0]DL_END[215, 1]DL_START[216, 0]DT_START;[216, 1]TEXTd [217, 2]DT_END[219, 0]DD_START:[219, 1]TEXTt [220, 2]INTLINK_START[[[222, 2]TEXTimage[224, 5]RESOURCESEP:[229, 1]TEXTfoo.jpg[230, 7]INTLINK_END]][237, 2]DD_END[239, 0]DL_END[239, 0]falsefalse[239, 0]

data/test/data/lex6

@@ -1,8 +1,8 @@
-PARA_STARTTEXTThis is some text with BOLDSTART'''TEXTboldBOLDEND'''TEXT and ITALICSTART''TEXTitalicITALICEND''TEXT formating.
+PARA_START[0, 0]TEXTThis is some text with [0, 23]BOLD_START'''[23, 3]TEXTbold[26, 4]BOLD_END'''[30, 3]TEXT and [33, 5]ITALIC_START''[38, 2]TEXTitalic[40, 6]ITALIC_END''[46, 2]TEXT formating.
 The list is also here:
-PARA_ENDUL_STARTLI_STARTTEXTList Item 1
-UL_STARTLI_STARTTEXTSub list item 1
-LI_ENDLI_STARTTEXTSub list item 2
-LI_ENDUL_ENDLI_ENDLI_STARTTEXTList Item 2
-LI_ENDUL_ENDPARA_STARTTEXTThe end
-PARA_ENDHLINE----falsefalse
+[48, 35]PARA_END[83, 0]UL_START[83, 0]LI_START[83, 1]TEXT List Item 1
+[84, 13]UL_START[97, 0]LI_START[97, 2]TEXT Sub list item 1
+[99, 17]LI_END[116, 0]LI_START[116, 2]TEXT Sub list item 2
+[118, 17]LI_END[135, 0]UL_END[135, 0]LI_END[135, 0]LI_START[135, 1]TEXT List Item 2
+[136, 13]LI_END[149, 0]UL_END[149, 0]PARA_START[149, 0]TEXTThe end
+[149, 8]PARA_END[157, 0]HLINE----[157, 4]falsefalse[162, 0]

data/test/data/lex7

@@ -1,35 +1,59 @@
-PARA_STARTLINKSTARTTEXThttp://sun.comLINKEND]PARA_END
+PARA_START[0, 0]LINK_START[0, 0]TEXThttp://sun.com[0, 14]LINK_END[14, 0]TEXT
 
-PARA_STARTLINKSTART[TEXThttp://sun.comLINKEND]PARA_END
+[14, 2]PARA_END[16, 0]PARA_START[16, 0]LINK_START[16, 0]TEXThttp://sun.com:80[16, 17]LINK_END[33, 0]TEXT
 
-PARA_STARTLINKSTART[TEXTmailto:joe@sun.comLINKEND]PARA_END
+[33, 2]PARA_END[35, 0]PARA_START[35, 0]LINK_START[[35, 1]TEXThttp://sun.com[36, 14]LINK_END][50, 1]TEXT
 
-PARA_STARTLINKSTART[TEXThttp://sun.comLINKSEPTEXTstarsLINKEND]PARA_END
+[51, 2]PARA_END[53, 0]PARA_START[53, 0]LINK_START[[53, 1]TEXTmailto:joe@sun.com[54, 18]LINK_END][72, 1]TEXT
 
-PARA_STARTLINKSTART[TEXThttp://sun.comLINKSEPTEXTstars and moonLINKEND]PARA_END
+[73, 2]PARA_END[75, 0]PARA_START[75, 0]LINK_START[[75, 1]TEXThttp://sun.com[76, 14]LINKSEP [90, 1]TEXTstars[91, 5]LINK_END][96, 1]TEXT
 
-PARA_STARTLINKSTART[TEXThttp://sun.comLINKSEPTEXTstars and ITALICSTART''TEXTmoonITALICEND''LINKSEPTEXTand BOLDSTART'''TEXTtreesBOLDEND'''LINKSEPTEXTand birdsLINKEND]PARA_END
+[97, 2]PARA_END[99, 0]PARA_START[99, 0]LINK_START[[99, 1]TEXThttp://sun.com[100, 14]LINKSEP [114, 1]TEXTstars and moon[115, 14]LINK_END][129, 1]TEXT
 
-PARA_STARTINTLINKSTART[[TEXTsunINTLINKEND]]PARA_END
+[130, 2]PARA_END[132, 0]PARA_START[132, 0]LINK_START[[132, 1]TEXThttp://sun.com[133, 14]LINKSEP [147, 1]TEXTstars and [148, 10]ITALIC_START''[158, 2]TEXTmoon[160, 4]ITALIC_END''[164, 2]TEXT and [166, 5]BOLD_START'''[171, 3]TEXTtrees[174, 5]BOLD_END'''[179, 3]TEXT and birds[182, 10]LINK_END][192, 1]TEXT
 
-PARA_STARTINTLINKSTART[[TEXTsunINTLINKSEP|TEXTAll about SunINTLINKEND]]PARA_END
+[193, 2]PARA_END[195, 0]PARA_START[195, 0]LINK_START[[195, 1]TEXThttp://sun.com[196, 14]LINKSEP [210, 1]TAG_STARTtt[211, 4]TEXTsun[215, 3]TAG_ENDtt[218, 5]TEXT and [223, 5]TAG_STARTtt[228, 4]TEXTmoon[232, 4]TAG_ENDtt[236, 5]LINK_END][241, 1]TEXT
 
-PARA_STARTINTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTsunINTLINKSEP|TEXTAll about SunINTLINKEND]]PARA_END
+[242, 2]PARA_END[244, 0]PARA_START[244, 0]LINK_START[[244, 1]TEXThttp://sun.com[245, 14]LINKSEP [259, 1]TEXTsun > moon[260, 10]LINK_END][270, 1]TEXT
 
-PARA_STARTINTLINKSTART[[TEXTsunINTLINKSEP|TEXTnofollow|All about SunINTLINKEND]]PARA_END
+[271, 2]PARA_END[273, 0]PARA_START[273, 0]INTLINK_START[[[273, 2]TEXTsun[275, 3]INTLINK_END]][278, 2]TEXT
 
-PARA_STARTINTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTsunINTLINKSEP|TEXTnofollowINTLINKSEP|TEXTAll about SunINTLINKEND]]PARA_END
+[280, 2]PARA_END[282, 0]PARA_START[282, 0]INTLINK_START[[[282, 2]TEXTsun[284, 3]INTLINKSEP|[287, 1]TEXTAll about Sun[288, 13]INTLINK_END]][301, 2]TEXT
 
-PARA_STARTINTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTsunINTLINKSEP|TEXTAll about INTLINKSTART[[TEXTSunINTLINKEND]]INTLINKEND]]PARA_END
+[303, 2]PARA_END[305, 0]PARA_START[305, 0]INTLINK_START[[[305, 2]TEXTimage[307, 5]RESOURCESEP:[312, 1]TEXTsun[313, 3]INTLINKSEP|[316, 1]TEXTAll about Sun[317, 13]INTLINK_END]][330, 2]TEXT
 
-PARA_STARTINTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTsunINTLINKSEP|TEXTAll about INTLINKSTART[[TEXTsunINTLINKSEP|TEXTMore aboutINTLINKEND]]INTLINKEND]]PARA_END
+[332, 2]PARA_END[334, 0]PARA_START[334, 0]INTLINK_START[[[334, 2]TEXTsun[336, 3]INTLINKSEP|[339, 1]TEXTnofollow|All about Sun[340, 22]INTLINK_END]][362, 2]TEXT
 
-PARA_STARTINTLINKSTART[[TEXTimageRESOURCE_SEP:TEXTsunINTLINKSEP|TEXToneINTLINKSEP|INTLINKSEP|TEXTtwoINTLINKSEP|INTLINKSEP|TEXTthreeINTLINKSEP|INTLINKEND]]PARA_END
+[364, 2]PARA_END[366, 0]PARA_START[366, 0]INTLINK_START[[[366, 2]TEXTsun[368, 3]INTLINKSEP|[371, 1]TEXTnofollow|Sun > Moon[372, 19]INTLINK_END]][391, 2]TEXT
 
-PARA_STARTTEXT[]PARA_END
+[393, 2]PARA_END[395, 0]PARA_START[395, 0]INTLINK_START[[[395, 2]TEXTsun[397, 3]INTLINKSEP|[400, 1]TEXTnofollow|[401, 9]TAG_STARTtt[410, 4]TEXTsun[414, 3]TAG_ENDtt[417, 5]TEXT and [422, 5]TAG_STARTtt[427, 4]TEXTmoon[431, 4]TAG_ENDtt[435, 5]INTLINK_END]][440, 2]TEXT
 
-PARA_STARTTEXT[ ]PARA_END
+[442, 2]PARA_END[444, 0]PARA_START[444, 0]INTLINK_START[[[444, 2]TEXTimage[446, 5]RESOURCESEP:[451, 1]TEXTsun[452, 3]INTLINKSEP|[455, 1]TEXTnofollow[456, 8]INTLINKSEP|[464, 1]TEXTAll about Sun[465, 13]INTLINK_END]][478, 2]TEXT
 
-PARA_STARTTEXT[[]]PARA_END
+[480, 2]PARA_END[482, 0]PARA_START[482, 0]INTLINK_START[[[482, 2]TEXTimage[484, 5]RESOURCESEP:[489, 1]TEXTsun[490, 3]INTLINKSEP|[493, 1]TEXTAll about [494, 10]INTLINK_START[[[504, 2]TEXTSun[506, 3]INTLINK_END]][509, 2]INTLINK_END]][511, 2]TEXT
 
-PARA_STARTTEXT[[ ]]PARA_ENDfalsefalse
+[513, 2]PARA_END[515, 0]PARA_START[515, 0]INTLINK_START[[[515, 2]TEXTimage[517, 5]RESOURCESEP:[522, 1]TEXTsun[523, 3]INTLINKSEP|[526, 1]TEXTAll about [527, 10]INTLINK_START[[[537, 2]TEXTsun[539, 3]INTLINKSEP|[542, 1]TEXTMore about[543, 10]INTLINK_END]][553, 2]INTLINK_END]][555, 2]TEXT
+
+[557, 2]PARA_END[559, 0]PARA_START[559, 0]INTLINK_START[[[559, 2]TEXTimage[561, 5]RESOURCESEP:[566, 1]TEXTsun[567, 3]INTLINKSEP|[570, 1]TEXTone[571, 3]INTLINKSEP|[574, 1]INTLINKSEP|[575, 1]TEXTtwo[576, 3]INTLINKSEP|[579, 1]INTLINKSEP|[580, 1]TEXTthree[581, 5]INTLINKSEP|[586, 1]INTLINK_END]][587, 2]TEXT
+
+[589, 2]PARA_END[591, 0]PARA_START[591, 0]INTLINK_START[[[591, 2]TEXTimage[593, 5]RESOURCESEP:[598, 1]TEXTsun[599, 3]INTLINKSEP|[602, 1]TEXTone[603, 3]INTLINKSEP|[606, 1]INTLINK_START[[[607, 2]TEXTaaa[609, 3]INTLINKSEP|[612, 1]TEXTbbb|ccc[613, 7]INTLINK_END]][620, 2]INTLINKSEP|[622, 1]INTLINK_START[[[623, 2]TEXTimage[625, 5]RESOURCESEP:[630, 1]TEXTmoon[631, 4]INTLINKSEP|[635, 1]TEXTaaa[636, 3]INTLINKSEP|[639, 1]TEXTbbb[640, 3]INTLINKSEP|[643, 1]TEXTccc[644, 3]INTLINK_END]][647, 2]INTLINKSEP|[649, 1]TEXTtwo[650, 3]INTLINK_END]][653, 2]TEXT
+
+[655, 2]PARA_END[657, 0]PARA_START[657, 0]TEXT[]
+
+[657, 4]PARA_END[661, 0]PARA_START[661, 0]TEXT[ ]
+
+[661, 5]PARA_END[666, 0]PARA_START[666, 0]TEXT[[]]
+
+[666, 6]PARA_END[672, 0]PARA_START[672, 0]INTLINK_START[[[672, 2]TEXT [674, 1]INTLINK_END]][675, 2]TEXT
+
+[677, 2]PARA_END[679, 0]PARA_START[679, 0]INTLINK_START[[[679, 2]CATEGORYCategory:[681, 9]TEXTCategory[690, 8]INTLINK_END]][698, 2]TEXT
+
+[700, 2]PARA_END[702, 0]PARA_START[702, 0]INTLINK_START[[[702, 2]CATEGORYCategory:[704, 9]TEXTCategory name[713, 13]INTLINK_END]][726, 2]TEXT
+
+[728, 2]PARA_END[730, 0]PARA_START[730, 0]INTLINK_START[[[730, 2]CATEGORYCategory:[732, 9]TEXTCategory name[741, 13]INTLINKSEP|[754, 1]TEXTSort as[755, 7]INTLINK_END]][762, 2]TEXT
+
+[764, 2]PARA_END[766, 0]PARA_START[766, 0]INTLINK_START[[[766, 2]RESOURCESEP:[768, 1]CATEGORYCategory:[769, 9]TEXTCategory[778, 8]INTLINK_END]][786, 2]TEXT
+
+[788, 2]PARA_END[790, 0]PARA_START[790, 0]INTLINK_START[[[790, 2]RESOURCESEP:[792, 1]CATEGORYCategory:[793, 9]TEXTCategory name[802, 13]INTLINK_END]][815, 2]TEXT
+
+[817, 2]PARA_END[819, 0]PARA_START[819, 0]INTLINK_START[[[819, 2]RESOURCESEP:[821, 1]CATEGORYCategory:[822, 9]TEXTCategory name[831, 13]INTLINKSEP|[844, 1]TEXTFoo[845, 3]INTLINK_END]][848, 2]PARA_END[850, 0]falsefalse[850, 0]

data/test/data/lex8

@@ -1 +1 @@
-SECTION_START===TEXT foo SECTION_END===falsefalse
+SECTION_START===[0, 3]TEXT foo [3, 5]SECTION_END===[8, 3]falsefalse[11, 0]

data/test/data/lex9

@@ -1,14 +1,14 @@
-PARA_STARTTEXTSome examples of wiki escaping.PARA_END
+PARA_START[0, 0]TEXTSome examples of wiki escaping.
 
-PARA_STARTTEXTA paragraph with some wiki characters: ''italic'' '''bold''' [[Link]]PARA_END
+[0, 33]PARA_END[33, 0]PARA_START[33, 0]TEXTA paragraph with some wiki characters: ''italic'' '''bold''' [[Link]]
 
-PARA_STARTTEXTAn escaped list:
+[33, 71]PARA_END[121, 0]PARA_START[121, 0]TEXTAn escaped list:
 
 * one
 * two
 * three
-PARA_END
 
-PARA_STARTTEXTText escaped in a heading:PARA_END
 
-SECTION_START==TEXTHeading with '''SECTION_END==falsefalse
+[121, 40]PARA_END[178, 0]PARA_START[178, 0]TEXTText escaped in a heading:
+
+[178, 28]PARA_END[206, 0]SECTION_START==[206, 2]TEXTHeading with '''[208, 16]SECTION_END==[241, 3]falsefalse[244, 0]

data/test/dataproducers/ast.rb

@@ -0,0 +1,24 @@
+require 'mediacloth/mediawikilexer'
+require 'mediacloth/mediawikiparser'
+require 'debugwalker'
+
+def produce(index)
+    file = File.new("#{File.dirname(__FILE__)}/../data/ast#{index}", "w")
+    inputFile = File.new("#{File.dirname(__FILE__)}/../data/input#{index}", "r")
+    input = inputFile.read
+
+    parser = MediaWikiParser.new
+    parser.lexer = MediaWikiLexer.new
+    ast = parser.parse(input)
+    walker = DebugWalker.new
+    walker.parse(ast)
+    
+    file.write(walker.tree)
+    file.close
+end
+
+if ARGV.empty?
+    (1..21).each { |i| produce(i) }
+else
+    produce ARGV[0].to_i
+end

data/test/dataproducers/html.rb

@@ -1,24 +1,23 @@
-require 'mediacloth/mediawikilexer'
-require 'mediacloth/mediawikiparser'
-require 'mediacloth/mediawikiparams'
-require 'mediacloth/mediawikihtmlgenerator'
+require 'mediacloth'
 
 def produce(index)
-    file = File.new("../test/data/html#{index}", "w")
-    inputFile = File.new("../test/data/input#{index}", "r")
+    file = File.new("#{File.dirname(__FILE__)}/../data/html#{index}", "w")
+    inputFile = File.new("#{File.dirname(__FILE__)}/../data/input#{index}", "r")
     input = inputFile.read
-
     parser = MediaWikiParser.new
     parser.lexer = MediaWikiLexer.new
     ast = parser.parse(input)
-    MediaWikiParams.instance.time = Time.mktime(2000, 1, 1, 1, 1, 1, 1)
+    params = MediaWikiParams.new
+    params.time = Time.mktime(2000, 1, 1, 1, 1, 1, 1)
     generator = MediaWikiHTMLGenerator.new
+    generator.params = params
     generator.parse(ast)
-
-#    puts generator.html
     file.write(generator.html)
     file.close
 end
 
-# (3..5).each { |i| produce(i) }
-produce(5)
+if ARGV.empty?
+    (1..13).each { |i| produce(i) }
+else
+    produce ARGV[0].to_i
+end

data/test/dataproducers/lex.rb

@@ -1,8 +1,10 @@
 require 'mediacloth/mediawikilexer'
 
+puts 
+
 def produce(index)
-    file = File.new("../test/data/lex#{index}", "w")
-    inputFile = File.new("../test/data/input#{index}", "r")
+    file = File.new("#{File.dirname(__FILE__)}/../data/lex#{index}", "w")
+    inputFile = File.new("#{File.dirname(__FILE__)}/../data/input#{index}", "r")
     input = inputFile.read
 
     lexer = MediaWikiLexer.new
@@ -11,5 +13,8 @@ def produce(index)
     file.close
 end
 
-#1..5.each { |i| produce(1) }
-produce(5)
+if ARGV.empty?
+    (1..13).each { |i| produce(i) }
+else
+    produce ARGV[0].to_i
+end

data/test/debugwalker.rb

@@ -17,18 +17,16 @@ class DebugWalker < MediaWikiWalker
     def initialize
         super
         @tree = ""
+        @left_space = 0
     end
 
 protected
 
     def parse_wiki_ast(ast)
         info(ast)
-        super(ast)
-    end
-
-    def parse_paragraph(ast)
-        info(ast)
-        super(ast)
+        @left_space += 1
+        super(ast) if ast.children
+        @left_space -= 1
     end
 
     def parse_text(ast)
@@ -38,26 +36,42 @@ protected
 
     def parse_list(ast)
         info(ast, ast.class)
+        @left_space += 1
         super(ast)
+        @left_space -= 1
     end
 
-    def parse_list_item(ast)
+    def parse_internal_link(ast)
         info(ast)
-        super(ast)
+        @left_space += 1
+        ast.children.map do |c|
+            parse_wiki_ast(c)
+        end
+        @left_space -= 1
     end
 
-    def parse_preformatted(ast)
+    def parse_resource_link(ast)
         info(ast)
+        @left_space += 1
+        ast.children.map do |c|
+            parse_wiki_ast(c)
+        end
+        @left_space -= 1
     end
 
-    def parse_section(ast)
+    def parse_category_link(ast)
         info(ast)
+        @left_space += 1
+        ast.children.map do |c|
+            parse_wiki_ast(c)
+        end
+        @left_space -= 1
     end
 
 private
     #Pretty-print ast node information
     def info(ast, *args)
-        @tree += "#{ast.class}"
+        @tree += "#{"    " * @left_space}#{ast.class}[#{ast.index}, #{ast.length}]"
         if args.length > 0
             @tree += ": "
             args.each { |arg| @tree += "#{arg} " }

data/test/htmlgenerator.rb

@@ -1,26 +1,183 @@
 require 'mediacloth/mediawikilexer'
 require 'mediacloth/mediawikiparser'
 require 'mediacloth/mediawikihtmlgenerator'
+require 'mediacloth/mediawikilinkhandler'
+require 'mediacloth/mediawikitemplatehandler'
 
 require 'test/unit'
 require 'testhelper'
 
 class HTMLGenerator_Test < Test::Unit::TestCase
 
-    include TestHelper
+    class << self
+      include TestHelper
+    end
+
+    test_files("html") do |input,result, name|
+      name =~ /([0-9]+)$/
+      define_method("test_input_#{$1}") do
+        assert_generates(result, input, nil, name)
+      end
+    end
+
+    def test_http_urls_in_internal_links
+      html = generate('[[http://www.google.com]]', MediaWikiLinkHandler.new)
+      assert_equal html, '<p><a href="http://www.google.com">http://www.google.com</a></p>'
+      html = generate('[[https://www.google.com]]', MediaWikiLinkHandler.new)
+      assert_equal html, '<p><a href="https://www.google.com">https://www.google.com</a></p>'
+    end
+
+    def test_punctuation_at_the_end_of_absolute_links
+      html = generate('http://example.com.', MediaWikiLinkHandler.new)
+      assert_equal html, '<p><a href="http://example.com">http://example.com</a>.</p>'
+      html = generate('http://example.com...', MediaWikiLinkHandler.new)
+      assert_equal html, '<p><a href="http://example.com">http://example.com</a>...</p>'
+      html = generate('http://example.com,', MediaWikiLinkHandler.new)
+      assert_equal html, '<p><a href="http://example.com">http://example.com</a>,</p>'
+      html = generate('http://example.com:', MediaWikiLinkHandler.new)
+      assert_equal html, '<p><a href="http://example.com">http://example.com</a>:</p>'
+      html = generate('http://example.com;', MediaWikiLinkHandler.new)
+      assert_equal html, '<p><a href="http://example.com">http://example.com</a>;</p>'
+      html = generate('http://example.com-', MediaWikiLinkHandler.new)
+      assert_equal html, '<p><a href="http://example.com">http://example.com</a>-</p>'
+    end
+
+    def test_punctuation_at_the_end_of_internal_links
+      html = generate('[http://example.com/page. Example]', MediaWikiLinkHandler.new)
+      assert_equal html, '<p><a href="http://example.com/page.">Example</a></p>'
+      html = generate('[http://example.com/page- Example]', MediaWikiLinkHandler.new)
+      assert_equal html, '<p><a href="http://example.com/page-">Example</a></p>'
+    end
+
+    def test_uses_element_attributes_from_link_handler
+      html = generate('[[InternalLink|This is just an internal link]]', ClassEmptyLinkHandler.new)
+      expected1 = '<p><a class="empty" href="http://www.example.com/wiki/InternalLink">This is just an internal link</a></p>'
+      expected2 = '<p><a href="http://www.example.com/wiki/InternalLink" class="empty">This is just an internal link</a></p>'
+      assert (html == expected1 or html == expected2)
+    end
+
+    def test_accepts_url_only_link_handlers
+      assert_generates '<p><a href="http://www.example.com/wiki/InternalLink/">This is just an internal link</a></p>',
+                       '[[InternalLink|This is just an internal link]]',
+                       UrlOnlyLinkHandler.new
+    end
+
+    def test_prefers_url_from_attributes_when_provided_with_ambiguous_link_info
+      html = generate('[[InternalLink|This is just an internal link]]', AmbiguousLinkHandler.new)
+      expected1 = '<p><a rel="nofollow" href="http://www.example.com/wiki/InternalLink">This is just an internal link</a></p>'
+      expected2 = '<p><a href="http://www.example.com/wiki/InternalLink" rel="nofollow">This is just an internal link</a></p>'
+      assert (html == expected1 or html == expected2)
+    end
+
+    def test_allows_specification_of_all_attributes
+      html = generate('[[MyLink|Here is my link]]', LinkAttributeHandler.new)
+      expected1 = '<p><a href="http://www.mysite.com/MyLink" id="123">Here is my link</a></p>'
+      expected2 = '<p><a id="123" href="http://www.mysite.com/MyLink">Here is my link</a></p>'
+      assert (html == expected1 or html == expected2)
+    end
+
+    def test_allows_full_customization_of_link_tags
+      assert_generates '<p><span class="link">This doesn\'t even render into a real link</span></p>',
+                       "[[AnotherLink|This doesn't even render into a real link]]",
+                        FullLinkHandler.new
+    end
+
+    def test_handles_category_links
+      assert_generates '<p><a class="category" href="http://www.mywiki.net/wiki/Category:Foo">More articles on Foo</a></p>',
+                       "[[:Category:Foo|More articles on Foo]]",
+                        CategoryLinkHandler.new
+    end
+
+    def test_handles_category_directives
+      assert_generates '<p><div id="catlinks">This page belongs to the <a class="category" href="http://www.mywiki.net/wiki/Category:Foo">Foo category</a></div></p>',
+                       "[[Category:Foo]]",
+                        CategoryDirectiveHandler.new
+    end
+
+    def test_table_with_column_attributes
+      assert_generates "<table cellpadding=\"5\" border=\"1\"><tr><td>a\n</td></tr>\n<tr><td  colspan=\"4\" align=\"center\" style=\"background:#ffdead;\"> b\n</td></tr>\n</table>\n",
+        "{|\n|a\n|-\n| colspan=\"4\" align=\"center\" style=\"background:#ffdead;\"| b\n|}"
+    end
+
+    def test_table_with_column_attributes_cellspacing_and_border
+      assert_generates "<table cellpadding=\"10\" border=\"5\"><tr><td>a\n</td></tr>\n<tr><td  colspan=\"4\" align=\"center\" style=\"background:#ffdead;\"> b\n</td></tr>\n</table>\n",
+        "{|cellpadding=\"10\" border=\"5\"\n|a\n|-\n| colspan=\"4\" align=\"center\" style=\"background:#ffdead;\"| b\n|}"
+    end
 
-    def test_input
-        test_files("html") { |input,result,name|
-            puts name
-            parser = MediaWikiParser.new
-            parser.lexer = MediaWikiLexer.new
-            ast = parser.parse(input)
-            MediaWikiParams.instance.time = Time.utc(2000, 1, 1, 1, 1, 1, 1)
-            generator = MediaWikiHTMLGenerator.new
-            generator.parse(ast)
-            assert_equal generator.html, result, name
-#             puts generator.html
-        }
+    def test_table_with_column_attributes_cellspacing_and_border_and_style
+      assert_generates "<table cellspacing=\"0\" cellpadding=\"5\" style=\"border-collapse:collapse\"><tr><td>a\n</td></tr>\n<tr><td  colspan=\"4\" align=\"center\" style=\"background:#ffdead;\"> b\n</td></tr>\n</table>\n",
+        "{|cellspacing=\"0\" cellpadding=\"5\" style=\"border-collapse:collapse\"\n|a\n|-\n| colspan=\"4\" align=\"center\" style=\"background:#ffdead;\"| b\n|}"
     end
 
+    def test_table_without_column_attributes
+      assert_generates "<table cellpadding=\"5\" border=\"1\"><tr><td> a\n</td><td> b\n</td></tr>\n</table>\n",
+        "{|\n| a\n| b\n|}"
+    end
+
+private
+
+  def assert_generates(result, input, link_handler=nil, message=nil)
+      assert_equal(result, generate(input, link_handler), message)
+   end
+
+   def generate(input, link_handler = nil)
+      parser = MediaWikiParser.new
+      parser.lexer = MediaWikiLexer.new
+      ast = parser.parse(input)
+      generator = MediaWikiHTMLGenerator.new
+      generator.link_handler = link_handler if link_handler
+      params = MediaWikiParams.new
+      params.time = Time.utc(2000, 1, 1, 1, 1, 1, 1)
+      generator.params = params
+      generator.parse(ast)
+      generator.html
+   end
+end
+
+class LinkAttributeHandler < MediaWikiLinkHandler
+  def link_attributes_for(page)
+    { :href => "http://www.mysite.com/#{page}", :id => '123' }
+  end
+end
+
+class ClassEmptyLinkHandler < MediaWikiLinkHandler
+  def link_attributes_for(resource)
+    {:class => 'empty', :href => "http://www.example.com/wiki/#{resource}"}
+  end
+end
+
+class UrlOnlyLinkHandler < MediaWikiLinkHandler
+  def url_for(resource)
+    "http://www.example.com/wiki/#{resource}/"
+  end
+end
+
+class AmbiguousLinkHandler < MediaWikiLinkHandler
+  def url_for(resource)
+    "http://www.somewhereelse.net"
+  end
+
+  def link_attributes_for(resource)
+    {:rel => 'nofollow', :href => "http://www.example.com/wiki/#{resource}"}
+  end
+end
+
+class FullLinkHandler < MediaWikiLinkHandler
+  def link_for(page, text)
+    "<span class=\"link\">#{text}</span>"
+  end
 end
+
+class CategoryLinkHandler 
+  def link_for_category(locator, text)
+    %(<a class="category" href="http://www.mywiki.net/wiki/Category:#{locator}">#{text}</a>)
+  end
+end
+
+class CategoryDirectiveHandler
+  def category_add(name, sort)
+    %(<div id="catlinks">This page belongs to the <a class="category" href="http://www.mywiki.net/wiki/Category:#{name}">#{name} category</a></div>)
+  end
+end
+
+