maruku 0.2.13 → 0.3.0

data/lib/maruku/string_utils.rb

@@ -16,11 +16,19 @@
 #   along with Maruku; if not, write to the Free Software
 #   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
+module MarukuStrings
+	def add_tabs(s,n=1,char="\t")
+		s.split("\n").map{|x| char*n+x }.join("\n")
+	end
+end
+class MDElement
+	include MarukuStrings
+end
 
 class Maruku
 	
 	# Split a string into lines, and chomps the newline
-	def split_lines(s)
+	def Maruku.split_lines_old(s)
 		a = []
 		s.each_line do |l| 
 			l = l.chomp
@@ -28,6 +36,10 @@ class Maruku
 		end	
 		a
 	end
+
+	def Maruku.split_lines(s)
+		s.split("\n")
+	end
 	
 	# This parses email headers. Returns an hash. hash['data'] is the message.
 	def parse_email_headers(s)
@@ -121,7 +133,7 @@ class Maruku
 			while s[i,1] =~ /\s/; i+=1 end
 			return i
 		else
-			$stderr.puts "Error: #{s} is not a list"
+			tell_user "BUG (my bad): '#{s}' is not a list"
 			0
 		end
 	end
@@ -164,15 +176,10 @@ class Maruku
 		s[i, s.size-1]
 	end
 
-
-	def debug(s)
-		$stderr.puts s
-	end
-
 	def dbg_describe_ary(a, prefix='')
 		i = 0 
 		a.each do |l|
-			$stderr.puts "#{prefix} (#{i+=1})##{l}#"
+			tell_user "#{prefix} (#{i+=1})##{l}#"
 		end
 	end
 
@@ -184,7 +191,7 @@ class Maruku
 		# raw html is like PHP Markdown Extra: at most three spaces before
 		return :code     if number_of_leading_spaces(l)>=4
 		return :footnote_text      if l =~ FootnoteText
-		return :ref if l =~ LinkRegex or l=~ IncompleteLink
+		return :ref_definition if l =~ LinkRegex or l=~ IncompleteLink
 		return :abbreviation if l =~ Abbreviation
 		return :definition if l =~ Definition
 		# I had a bug with emails and urls at the beginning of the 
@@ -244,7 +251,7 @@ class Maruku
 	# This regex is taken from BlueCloth sources
 	# Link defs are in the form: ^[id]: \n? url "optional title"
 	LinkRegex = %r{
-		^[ ]*\[(.+)\]:		# id = $1
+		^[ ]*\[([^\]]+)\]:		# id = $1
 		  [ ]*
 		<?(\S+)>?				# url = $2
 		  [ ]*

data/lib/maruku/structures.rb

@@ -19,7 +19,7 @@
 
 
 # I did not want to have a class for each possible element. 
-# Instead I opted to have the class "MDElement"
+# Instead I opted to have only the class "MDElement"
 # that represents eveything in the document (paragraphs, headers, etc).
 #
 # You can tell what it is by the variable `node_type`. 
@@ -44,7 +44,7 @@
 #
 #     @doc.meta[:my_property] # => 'value'
 #
-# from whichever MDElement.
+# from whichever MDElement in the hierarchy.
 
 class MDElement 
 	# XXX List not complete
@@ -58,19 +58,74 @@ class MDElement
 	# :li :want_my_paragraph
 	#  :header: :level
 	# code, inline_code: :raw_code
-	attr_accessor :meta
+	attr_reader :meta
 	# reference of containing document (document has list of ref)
 	attr_accessor :doc
 	
-	def initialize
+	def initialize(node_type_=:unset, children_=[], meta_={} )
 		super(); 
-		@children = []; 
-		@node_type = :unset
-		@meta = {};
+		raise 'children is nil' if not children_
+		raise 'meta is nil' if not meta_
+		
+		@children = children_
+		@node_type = node_type_
+		@meta = meta_
+	end
+	
+	def ==(o)
+		ok = o.kind_of?(MDElement) &&
+		(self.node_type == o.node_type) &&
+		(self.meta == o.meta) &&
+		(self.children == o.children)
+		ok
+	end
+	
+	def inspect(compact=true)
+		if compact
+			i2 = inspect2
+			return i2 if i2
+		end
+		
+		"md_el(:%s,%s %s)" %
+		[
+			@node_type,
+			children_inspect(compact), 
+			if @meta.size>0 then 
+				', '+@meta.inspect 
+			else '' end
+		]
+	end
+
+	def children_inspect(compact=true)
+		s = @children.inspect_more(compact,', ')
+		if @children.empty?
+			"[]"
+		elsif s.size < 70
+			s
+		else
+			"[\n"+
+			add_tabs(@children.inspect_more(compact,",\n ",false))+
+			"\n]"
+		end
 	end
 	
 end
 
+class String
+	alias inspect_more inspect
+end
+
+class Array
+	def inspect_more(compact, join_string, add_brackets=true)
+		s  = map {|x| 
+			x.kind_of?(String) ? x.inspect : 
+			x.kind_of?(MDElement) ? x.inspect(compact) : 
+			(raise "WTF #{x.class} #{x.inspect}")
+		}.join(join_string)
+		
+		add_brackets ? "[#{s}]" : s
+	end
+end
 # The Maruku class represent the whole document 
 # and holds global data.
 
@@ -78,32 +133,4 @@ class Maruku < MDElement
 	attr_accessor :refs
 	attr_accessor :footnotes
 	attr_accessor :abbreviations
-	# an instance of Section (see below)
-	attr_accessor :toc 
-end
-
-class MDElement
-	
-	# This represents a section in the TOC.
-	class Section
-		# a Fixnum, is == header_element.meta[:level]
-		attr_accessor :section_level 
-		
-		# An array of fixnum, like [1,2,5] for Section 1.2.5
-		attr_accessor :section_number 
-		
-		# reference to header (header has h.meta[:section] to self)
-		attr_accessor :header_element
-
-		# Array of immediate children of this element
-		attr_accessor :immediate_children
-		# Array of Section inside this section
-		attr_accessor :section_children
-		
-		
-		def initialize
-			@immediate_children = []
-			@section_children = []
-		end
-	end 
 end

data/lib/maruku/structures_iterators.rb

@@ -0,0 +1,39 @@
+
+
+
+class MDElement
+	
+	# yields to each element of specified node_type
+	def each_element(e_node_type, &block) 
+		@children.each do |c| 
+			if c.kind_of? MDElement
+				if (not e_node_type) || (e_node_type == c.node_type)
+					block.call c
+				end
+				c.each_element(e_node_type, &block)
+			end
+		end
+	end
+	
+	# Apply passed block to each String in the hierarchy.
+	def replace_each_string(&block)
+		for c in @children
+			if c.kind_of? MDElement
+				c.replace_each_string(&block)
+			end
+		end
+
+		processed = []
+		until @children.empty?
+			c = @children.shift
+			if c.kind_of? String
+				result = block.call(c)
+				[*result].each do |e| processed << e end
+			else
+				processed << c
+			end
+		end
+		@children = processed
+	end
+
+end

data/lib/maruku/tests/benchmark.rb

@@ -1,18 +1,26 @@
 require 'maruku'
 require 'bluecloth'
 
+
 data = $stdin.read
 
 num = 10
 
-stats = 
+methods = 
 [
 	
 	[BlueCloth, :to_html],
-	[Maruku,    :to_html],
-	[Maruku,    :to_latex]
+	[Maruku,    :class],
+#	[Maruku,    :to_html],
+#	[Maruku,    :to_latex]
 	
-].map do |c, method|
+]*2
+
+#methods = [[Maruku, :class]]
+#num = 10
+
+stats = 
+methods .map do |c, method|
 	puts "Computing for #{c}"
 
 	start = Time.now

data/lib/maruku/tests/new_parser.rb

@@ -0,0 +1,318 @@
+require 'maruku'
+require 'maruku/helpers'
+
+require 'maruku/parse_span_better'
+
+class Maruku
+
+
+	class TestNewParser
+		include Helpers
+		include MarukuErrors
+			
+		# 5 accented letters in italian, encoded as UTF-8
+		AccIta8 = "\303\240\303\250\303\254\303\262\303\271"
+
+		# Same letters, written in ISO-8859-1 (one byte per letter)
+		AccIta1 = "\340\350\354\362\371"
+		
+		# The word MA-RU-KU, written in katakana using UTF-8
+		Maruku8 = "\343\203\236\343\203\253\343\202\257"
+		
+		def do_it(verbose, break_on_first_error)
+		good_cases = [
+			["",       [],         'Empty string gives empty list'],
+			["a",      ["a"],      'Easy char'],
+			[' ',      [' '],      'One char => one string'],
+			['  ',     [' '],      'Two chars => one char'],
+			['a  b',   ['a b'],    'Spaces are compressed'],
+			['a  b',   ['a b'],    'Newlines are spaces'],
+			["a\nb",   ['a b'],    'Newlines are spaces'],
+			["a\n b",  ['a b'],    'Compress newlines 1'],
+			["a \nb",  ['a b'],    'Compress newlines 2'],
+			[" \nb",   [' b'],     'Compress newlines 3'],
+			["\nb",    [' b'],     'Compress newlines 4'],
+			["b\n",    ['b '],     'Compress newlines 5'],
+			["\n",     [' '],      'Compress newlines 6'],
+			["\n\n\n", [' '],      'Compress newlines 7'],
+			
+			[nil, :throw, "Should throw on nil input"],
+			
+			# Code blocks
+			["`" ,   :throw,  'Unclosed single ticks'],
+			["``" ,  :throw,  'Unclosed double ticks'],
+			["`a`" ,     [md_code('a')],    'Simple inline code'],
+			["`` ` ``" ,    [md_code('`')],   ],
+			["`` \\` ``" ,    [md_code('\\`')],   ],
+			["``a``" ,   [md_code('a')],    ],
+			["`` a ``" ,   [md_code('a')],    ],
+			
+			# Newlines 
+			["a  \n", ['a',md_el(:linebreak)], 'Two spaces give br.'],
+			["a \n",  ['a '], 'Newlines 2'],
+			["  \n",  [md_el(:linebreak)], 'Newlines 3'],
+			["  \n  \n",  [md_el(:linebreak),md_el(:linebreak)],'Newlines 3'],
+			["  \na  \n",  [md_el(:linebreak),'a',md_el(:linebreak)],'Newlines 3'],
+			
+			# Inline HTML
+			["a < b", ['a < b'], '< can be on itself'],
+			["<hr>",  [md_html('<hr />')], 'HR will be sanitized'],
+			["<hr/>", [md_html('<hr />')], 'Closed tag is ok'],
+			["<hr  />", [md_html('<hr />')], 'Closed tag is ok 2'],
+			["<hr/>a", [md_html('<hr />'),'a'], 'Closed tag is ok 2'],
+			["<em></em>a", [md_html('<em></em>'),'a'], 'Inline HTML 1'],
+			["<em>e</em>a", [md_html('<em>e</em>'),'a'], 'Inline HTML 2'],
+			["a<em>e</em>b", ['a',md_html('<em>e</em>'),'b'], 'Inline HTML 3'],
+			["<em>e</em>a<em>f</em>", 
+				[md_html('<em>e</em>'),'a',md_html('<em>f</em>')], 
+				'Inline HTML 4'],
+			["<em>e</em><em>f</em>a", 
+				[md_html('<em>e</em>'),md_html('<em>f</em>'),'a'], 
+				'Inline HTML 5'],
+				
+				["<img src='a' />", [md_html("<img src='a' />")], 'Attributes'],
+				["<img src='a'/>"],
+			
+			# emphasis
+			["**", :throw, 'Unclosed double **'],
+			["\\*", ['*'], 'Escaping of *'],
+			["a *b* ", ['a ', md_em('b'),' '], 'Emphasis 1'],
+			["a *b*", ['a ', md_em('b')], 'Emphasis 2'],
+			["a * b", ['a * b'], 'Emphasis 3'],
+			["a * b*", :throw, 'Unclosed emphasis'],
+			# same with underscore
+			["__", :throw, 'Unclosed double __'],
+			["\\_", ['_'], 'Escaping of _'],
+			["a _b_ ", ['a ', md_em('b'),' '], 'Emphasis 4'],
+			["a _b_", ['a ', md_em('b')], 'Emphasis 5'],
+			["a _ b", ['a _ b'], 'Emphasis 6'],
+			["a _ b_", :throw, 'Unclosed emphasis'],
+			["_b_", [md_em('b')], 'Emphasis 7'],
+			["_b_ _c_", [md_em('b'),' ',md_em('c')], 'Emphasis 8'],
+			["_b__c_", [md_em('b'),md_em('c')], 'Emphasis 9'],
+			# strong
+			["**a*", :throw, 'Unclosed double ** 2'],
+			["\\**a*", ['*', md_em('a')], 'Escaping of *'],
+			["a **b** ", ['a ', md_strong('b'),' '], 'Emphasis 1'],
+			["a **b**", ['a ', md_strong('b')], 'Emphasis 2'],
+			["a ** b", ['a ** b'], 'Emphasis 3'],
+			["a ** b**", :throw, 'Unclosed emphasis'],
+			["**b****c**", [md_strong('b'),md_strong('c')], 'Emphasis 9'],
+			# strong (with underscore)
+			["__a_", :throw, 'Unclosed double __ 2'],
+			["\\__a_", ['_', md_em('a')], 'Escaping of _'],
+			["a __b__ ", ['a ', md_strong('b'),' '], 'Emphasis 1'],
+			["a __b__", ['a ', md_strong('b')], 'Emphasis 2'],
+			["a __ b", ['a __ b'], 'Emphasis 3'],
+			["a __ b__", :throw, 'Unclosed emphasis'],
+			["__b____c__", [md_strong('b'),md_strong('c')], 'Emphasis 9'],
+			# extra strong
+			["***a**", :throw, 'Unclosed triple *** '],
+			["\\***a**", ['*', md_strong('a')], 'Escaping of *'],
+			["a ***b*** ", ['a ', md_emstrong('b'),' '], 'Strong elements'],
+			["a ***b***", ['a ', md_emstrong('b')]],
+			["a *** b", ['a *** b']],
+			["a ** * b", ['a ** * b']],
+			["***b******c***", [md_emstrong('b'),md_emstrong('c')]],
+			["a *** b***", :throw, 'Unclosed emphasis'],
+			# same with underscores
+			["___a__", :throw, 'Unclosed triple *** '],
+			["\\___a__", ['_', md_strong('a')], 'Escaping of *'],
+			["a ___b___ ", ['a ', md_emstrong('b'),' '], 'Strong elements'],
+			["a ___b___", ['a ', md_emstrong('b')]],
+			["a ___ b", ['a ___ b']],
+			["a __ _ b", ['a __ _ b']],
+			["___b______c___", [md_emstrong('b'),md_emstrong('c')]],
+			["a ___ b___", :throw, 'Unclosed emphasis'],
+			# mixing is bad
+			["*a_", :throw, 'Mixing is bad'],
+			["_a*", :throw],
+			["**a__", :throw],
+			["__a**", :throw],
+			["___a***", :throw],
+			["***a___", :throw],
+			# links of the form [text][ref]
+			["\\[a]",  ["[a]"], 'Escaping 1'],
+			["\\[a\\]", ["[a]"], 'Escaping 2'],
+			["[a]",   ["a"],   'Not a link'],
+			["[a][]",   [ md_link(["a"],'')], 'Empty link'],
+			["[a][]b",   [ md_link(["a"],''),'b'], 'Empty link'],
+			["[a\\]][]", [ md_link(["a]"],'')], 'Escape inside link'],
+			
+			["[a",  :throw,   'Link not closed'],
+			["[a][",  :throw,   'Ref not closed'],
+			
+			# links of the form [text](url)
+			["\\[a](b)",  ["[a](b)"], 'Links'],
+			["[a](url)c",  [md_im_link(['a'],'url'),'c'], 'url'],
+			["[a]( url )c" ],
+			["[a] (	url )c" ],
+			["[a] (	url)c" ],
+			
+			["[a](ur:/l/ 'Title')",  [md_im_link(['a'],'ur:/l/','Title')],
+			 	'url and title'],
+			["[a] (	ur:/l/ \"Title\")" ],
+			["[a] (	ur:/l/ \"Title\")" ],
+			["[a]( ur:/l/ Title)", :throw, "Must quote title" ],
+
+			["[a](url 'Tit\\\"l\\\\e')", [md_im_link(['a'],'url','Tit"l\\e')],
+			 	'url and title escaped'],
+			["[a] (	url \"Tit\\\"l\\\\e\")" ],
+			["[a] (	url	\"Tit\\\"l\\\\e\"  )" ],
+			['[a] (	url	"Tit\\"l\\\\e"  )' ],
+			["[a]()", [md_im_link(['a'],'')], 'No URL is OK'],
+		
+			["[a](\"Title\")", :throw, "No url specified" ],
+			["[a](url \"Title)", :throw, "Unclosed quotes" ],
+			["[a](url \"Title\\\")", :throw],
+			["[a](url \"Title\" ", :throw],
+
+			["[a](url \'Title\")", :throw, "Mixing is bad" ],
+			["[a](url \"Title\')"],
+			
+			["[a](/url)", [md_im_link(['a'],'/url')], 'Funny chars in url'],
+			["[a](#url)", [md_im_link(['a'],'#url')]],
+			["[a](</script?foo=1&bar=2>)", [md_im_link(['a'],'/script?foo=1&bar=2')]],
+			
+			
+			# Images
+			["\\![a](url)",  ['!', md_im_link(['a'],'url') ], 'Escaping images'],
+			
+			["![a](url)",  [md_im_image(['a'],'url')], 'Image no title'],
+			["![a]( url )" ],
+			["![a] (	url )" ],
+			["![a] (	url)" ],
+
+			["![a](url 'ti\"tle')",  [md_im_image(['a'],'url','ti"tle')], 'Image with title'],
+			['![a]( url "ti\\"tle")' ],
+
+			["![a](url", :throw, 'Invalid images'],
+			["![a( url )" ],
+			["![a] ('url )" ],
+
+			["![a][imref]",  [md_image(['a'],'imref')], 'Image with ref'],
+			["![a][ imref]"],
+			["![a][ imref ]"],
+			["![a][\timref\t]"],
+			
+			['<http://example.com/?foo=1&bar=2>', 
+				[md_url('http://example.com/?foo=1&bar=2')], 'Immediate link'],
+				['a<http://example.com/?foo=1&bar=2>b', 
+					['a',md_url('http://example.com/?foo=1&bar=2'),'b']  ],
+			['<andrea@censi.org>', 
+				[md_email('andrea@censi.org')], 'Email address'],
+			['<mailto:andrea@censi.org>'],
+					
+			["a<!-- -->b", ['a',md_html('<!-- -->'),'b'], 
+				'HTML Comment'],
+
+			["a<!--", :throw, 'Bad HTML Comment'],
+			["a<!-- ", :throw, 'Bad HTML Comment'],
+
+			["a <b", :throw, 'Bad HTML 1'],
+			["<b",   :throw, 'Bad HTML 2'],
+			["<b!",  :throw, 'Bad HTML 3'],
+			['`<div>`, `<table>`, `<pre>`, `<p>`',
+				[md_code('<div>'),', ',md_code('<table>'),', ',
+					md_code('<pre>'),', ',md_code('<p>')],
+					'Multiple HTLM tags'],
+					
+			["&andrea", ["&andrea"], 'Parsing of entities'],
+# no escaping is allowed
+#			["\\&andrea;", ["&andrea;"]],
+			["&&andrea;", ["&", md_entity('andrea')] ],
+			["&123;;&amp;",[md_entity('123'),';',md_entity('amp')]],
+			
+			["a\nThe [syntax page] [s] provides", 
+				['a The ', md_link(['syntax page'],'s'), ' provides'], 'Regression'],
+			
+			["#{Maruku8}", [Maruku8], "Reading UTF-8"],
+			["#{AccIta1}", [AccIta8], "Converting ISO-8859-1 to UTF-8", 
+				{:encoding => 'iso-8859-1'}],
+		]
+
+			count = 1; last_comment=""; last_expected=:throw
+			good_cases.each do |t|
+				if not t[1]
+					t[1] = last_expected
+				else
+					last_expected = t[1]
+				end				
+				if not t[2]
+					t[2] = last_comment + " #{count+=1}"
+				else
+					last_comment = t[2]; count=1
+				end
+			end
+			
+				
+			@verbose = verbose
+			m = Maruku.new
+			m.meta[:on_error] = :raise
+			
+			good_cases.each do |input, expected, comment|
+					output = nil
+					begin
+						output = m.parse_span_better(input)
+						#lines = Maruku.split_lines input
+						#output = m.parse_lines_as_span(lines)
+					rescue Exception => e
+						if not expected == :throw
+							ex = e.inspect+ "\n"+ e.backtrace.join("\n")
+							s = comment+describe_difference(input, expected, output)
+								
+							print_status(comment,'CRASHED :-(', ex+s)
+							raise e if @break_on_first_error 
+						else
+							print_status(comment,'OK')
+						end
+					end
+					
+					if not expected == :throw
+						if not (expected == output)
+							s = comment+describe_difference(input, expected, output)
+							print_status(comment, 'FAILED', s)
+							break if break_on_first_error
+						else
+							print_status(comment, 'OK')
+						end
+					else # I expected a raise
+						if output
+							s = comment+describe_difference(input, expected, output)
+							
+							print_status(comment, 'FAILED (no throw)', s)
+							break if break_on_first_error
+						end
+					end
+					
+			end
+		end
+		
+		PAD=40
+		def print_status(comment, status, verbose_text=nil)
+			if comment.size < PAD
+				comment = comment + (" "*(PAD-comment.size))
+			end
+			puts "- #{comment} #{status}"
+			if @verbose and verbose_text
+				puts verbose_text
+			end
+		end
+		
+		
+		def describe_difference(input, expected, output)
+			"\nInput:\n  #{input.inspect}" +
+			    "\nExpected:\n  #{expected.inspect}" +
+				"\nOutput:\n  #{output.inspect}\n"+
+				"\nExpected(more):\n  #{expected.inspect}" +
+				"\nOutput(more):\n    #{output.inspect}"
+		end
+		
+	end # class Test
+end
+
+verbose = ARGV.include? 'v'
+break_on_first = ARGV.include? 'b'
+Maruku::TestNewParser.new.do_it(verbose, break_on_first)
+
+