maiku 0.6.1.maiku

data/lib/maruku/input/charsource.rb

@@ -0,0 +1,326 @@
+#--
+#   Copyright (C) 2006  Andrea Censi  <andrea (at) rubyforge.org>
+#
+# This file is part of Maruku.
+# 
+#   Maruku is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License, or
+#   (at your option) any later version.
+# 
+#   Maruku is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+# 
+#   You should have received a copy of the GNU General Public License
+#   along with Maruku; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+
+
+module MaRuKu; module In; module Markdown; module SpanLevelParser
+
+# a string scanner coded by me
+class CharSourceManual; end
+
+# a wrapper around StringScanner
+class CharSourceStrscan; end
+
+# A debug scanner that checks the correctness of both
+# by comparing their output
+class CharSourceDebug; end
+
+# Choose!
+
+CharSource = CharSourceManual     # faster! 58ms vs. 65ms
+#CharSource = CharSourceStrscan
+#CharSource = CharSourceDebug
+
+
+class CharSourceManual
+	include MaRuKu::Strings
+	
+	def initialize(s, parent=nil)
+		raise "Passed #{s.class}" if not s.kind_of? String
+		@buffer = s
+		@buffer_index = 0
+		@parent = parent
+	end
+	
+	# Return current char as a FixNum (or nil).
+	def cur_char; @buffer[@buffer_index]   end
+
+	# Return the next n chars as a String.
+	def cur_chars(n); @buffer[@buffer_index,n]  end
+	
+	# Return the char after current char as a FixNum (or nil).
+	def next_char; @buffer[@buffer_index+1] end
+	
+	def shift_char
+		c = @buffer[@buffer_index]
+		@buffer_index+=1
+		c
+	end
+	
+	def ignore_char
+		@buffer_index+=1
+		nil
+	end
+	
+	def ignore_chars(n)
+		@buffer_index+=n
+		nil
+	end
+	
+	def current_remaining_buffer
+		@buffer[@buffer_index, @buffer.size-@buffer_index]
+	end
+	
+	def cur_chars_are(string)
+		# There is a bug here
+		if false
+			r2 = /^.{#{@buffer_index}}#{Regexp.escape string}/m
+			@buffer =~ r2
+		else
+			cur_chars(string.size) == string
+		end
+	end
+
+	def next_matches(r)
+		r2 = /^.{#{@buffer_index}}#{r}/m
+		md = r2.match @buffer
+		return !!md
+	end
+	
+	def read_regexp3(r)
+		r2 = /^.{#{@buffer_index}}#{r}/m
+		m = r2.match @buffer
+		if m
+			consumed = m.to_s.size - @buffer_index
+#			puts "Consumed #{consumed} chars (entire is #{m.to_s.inspect})"
+			ignore_chars consumed
+		else
+#			puts "Could not read regexp #{r2.inspect} from buffer "+
+#			" index=#{@buffer_index}"
+#			puts "Cur chars = #{cur_chars(20).inspect}"
+#			puts "Matches? = #{cur_chars(20) =~ r}"
+		end
+		m
+	end
+
+		def read_regexp(r)
+			r2 = /^#{r}/
+			rest = current_remaining_buffer
+			m = r2.match(rest)
+			if m
+				@buffer_index += m.to_s.size
+#				puts "#{r} matched #{rest.inspect}: #{m.to_s.inspect}"
+			end
+			return m
+		end
+	
+	def consume_whitespace
+		while c = cur_char 
+		  if (c == ?\s || c == ?\t)
+#				puts "ignoring #{c}"
+				ignore_char
+			else
+#				puts "#{c} is not ws: "<<c
+				break
+			end
+		end
+	end
+
+	def read_text_chars(out)
+		s = @buffer.size; c=nil
+		while @buffer_index < s && (c=@buffer[@buffer_index]) &&
+			 ((c>=?a && c<=?z) || (c>=?A && c<=?Z))
+				out << c
+				@buffer_index += 1
+		end
+	end
+	
+	def describe
+		s = describe_pos(@buffer, @buffer_index)
+		if @parent
+			s += "\n\n" + @parent.describe
+		end
+		s
+	end
+	include SpanLevelParser
+end
+
+def describe_pos(buffer, buffer_index)
+	len = 75
+	num_before = [len/2, buffer_index].min
+	num_after = [len/2, buffer.size-buffer_index].min
+	num_before_max = buffer_index
+	num_after_max = buffer.size-buffer_index
+	
+#		puts "num #{num_before} #{num_after}"
+	num_before = [num_before_max, len-num_after].min
+	num_after  = [num_after_max, len-num_before].min
+#		puts "num #{num_before} #{num_after}"
+	
+	index_start = [buffer_index - num_before, 0].max
+	index_end   = [buffer_index + num_after, buffer.size].min
+	
+	size = index_end- index_start
+	
+#		puts "- #{index_start} #{size}"
+
+	str = buffer[index_start, size]
+	str.gsub!("\n",'N')
+	str.gsub!("\t",'T')
+	
+	if index_end == buffer.size 
+		str += "EOF"
+	end
+		
+	pre_s = buffer_index-index_start
+	pre_s = [pre_s, 0].max
+	pre_s2 = [len-pre_s,0].max
+#		puts "pre_S = #{pre_s}"
+	pre =" "*(pre_s) 
+	
+	"-"*len+"\n"+
+	str + "\n" +
+	"-"*pre_s + "|" + "-"*(pre_s2)+"\n"+
+#		pre + "|\n"+
+	pre + "+--- Byte #{buffer_index}\n"+
+	
+	"Shown bytes [#{index_start} to #{size}] of #{buffer.size}:\n"+
+	add_tabs(buffer,1,">")
+	
+#		"CharSource: At character #{@buffer_index} of block "+
+#		" beginning with:\n    #{@buffer[0,50].inspect} ...\n"+
+#		" before: \n     ... #{cur_chars(50).inspect} ... "
+end
+
+
+require 'strscan'
+
+class CharSourceStrscan
+	include SpanLevelParser
+	include MaRuKu::Strings
+	
+	def initialize(s, parent=nil)
+		@s = StringScanner.new(s)
+		@parent = parent
+	end
+	
+	# Return current char as a FixNum (or nil).
+	def cur_char
+		 @s.peek(1)[0]
+	end
+
+	# Return the next n chars as a String.
+	def cur_chars(n); 
+		@s.peek(n)
+	end
+	
+	# Return the char after current char as a FixNum (or nil).
+	def next_char; 
+		@s.peek(2)[1]
+	end
+	
+	def shift_char
+		(@s.get_byte)[0]
+	end
+	
+	def ignore_char
+		@s.get_byte
+		nil
+	end
+	
+	def ignore_chars(n)
+		n.times do @s.get_byte end
+		nil
+	end
+	
+	def current_remaining_buffer
+		@s.rest #nil #@buffer[@buffer_index, @buffer.size-@buffer_index]
+	end
+	
+	def cur_chars_are(string)
+		cur_chars(string.size) == string
+	end
+
+	def next_matches(r)
+		len = @s.match?(r)
+		return !!len
+	end
+	
+	def read_regexp(r)
+		string = @s.scan(r)
+		if string
+			return r.match(string)
+		else
+			return nil
+		end
+	end
+	
+	def consume_whitespace
+		@s.scan(/\s+/)
+		nil
+	end
+	
+	def describe
+		describe_pos(@s.string, @s.pos)
+	end
+	
+end
+
+
+class CharSourceDebug
+	def initialize(s, parent)
+		@a = CharSourceManual.new(s, parent)
+		@b = CharSourceStrscan.new(s, parent)
+	end
+	
+	def method_missing(methodname, *args)
+		a_bef = @a.describe
+		b_bef = @b.describe
+		
+		a = @a.send(methodname, *args)
+		b = @b.send(methodname, *args)
+		
+#		if methodname == :describe
+#			return a
+#		end
+		
+		if a.kind_of? MatchData
+			if a.to_a != b.to_a
+				puts "called: #{methodname}(#{args})"
+				puts "Matchdata:\na = #{a.to_a.inspect}\nb = #{b.to_a.inspect}"
+				puts "AFTER: "+@a.describe
+				puts "AFTER: "+@b.describe
+				puts "BEFORE: "+a_bef
+				puts "BEFORE: "+b_bef
+				puts caller.join("\n")
+				exit
+			end
+		else
+			if a!=b
+				puts "called: #{methodname}(#{args})"
+				puts "Attenzione!\na = #{a.inspect}\nb = #{b.inspect}"
+				puts ""+@a.describe
+				puts ""+@b.describe
+				puts caller.join("\n")
+				exit
+			end
+		end
+		
+		if @a.cur_char != @b.cur_char
+			puts "Fuori sincronia dopo #{methodname}(#{args})"
+			puts ""+@a.describe
+			puts ""+@b.describe
+			exit
+		end
+		
+		return a
+	end
+end
+
+end end end end

data/lib/maruku/input/extensions.rb

@@ -0,0 +1,69 @@
+module MaRuKu; module In; module Markdown
+
+
+	# Hash Fixnum -> name
+	SpanExtensionsTrigger = {}
+	
+	
+	class SpanExtension
+		# trigging chars
+		attr_accessor :chars
+		# trigging regexp
+		attr_accessor :regexp
+		# lambda
+		attr_accessor :block
+	end
+	
+	# Hash String -> Extension
+	SpanExtensions = {}
+
+	def check_span_extensions(src, con)
+		c = src.cur_char
+		if extensions = SpanExtensionsTrigger[c]
+			extensions.each do |e|
+				if e.regexp && (match = src.next_matches(e.regexp))
+					return true if e.block.call(doc, src, con)
+				end
+			end
+		end
+		return false # not special
+	end
+	
+	def self.register_span_extension(args)
+		e = SpanExtension.new
+		e.chars = [*args[:chars]]
+		e.regexp = args[:regexp]
+		e.block = args[:handler] || raise("No blocks passed")
+		e.chars.each do |c|
+			(SpanExtensionsTrigger[c] ||= []).push e
+		end
+	end
+
+	def self.register_block_extension(args)
+		regexp = args[:regexp]
+		BlockExtensions[regexp] = (args[:handler] || raise("No blocks passed"))
+	end
+
+	# Hash Regexp -> Block
+	BlockExtensions = {}
+
+	def check_block_extensions(src, con, line)
+		BlockExtensions.each do |reg, block|
+			if m = reg.match(line)
+				block = BlockExtensions[reg]
+				accepted =  block.call(doc, src, con)
+				return true if accepted
+			end
+		end
+		return false # not special
+	end
+	
+	def any_matching_block_extension?(line)
+		BlockExtensions.each_key do |reg|
+			m = reg.match(line)
+			return m if m
+		end
+		return false
+	end
+	
+end end end

data/lib/maruku/input/html_helper.rb

@@ -0,0 +1,189 @@
+#--
+#   Copyright (C) 2006  Andrea Censi  <andrea (at) rubyforge.org>
+#
+# This file is part of Maruku.
+# 
+#   Maruku is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License, or
+#   (at your option) any later version.
+# 
+#   Maruku is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+# 
+#   You should have received a copy of the GNU General Public License
+#   along with Maruku; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+
+
+module MaRuKu; module In; module Markdown; module SpanLevelParser
+
+# This class helps me read and sanitize HTML blocks
+
+# I tried to do this with REXML, but wasn't able to. (suggestions?)
+
+	class HTMLHelper
+		include MaRuKu::Strings
+		
+		Tag = %r{^<(/)?(\w+)\s*([^>]*)>}m
+		PartialTag = %r{^<.*}m
+
+		EverythingElse = %r{^[^<]+}m
+		CommentStart = %r{^<!--}x
+		CommentEnd = %r{^.*-->}
+		TO_SANITIZE = ['img','hr','br'] 
+		
+		attr_reader :rest
+		
+		def my_debug(s)
+#			puts "---"*10+"\n"+inspect+"\t>>>\t"s
+		end
+		
+		def initialize 
+			@rest = ""
+			@tag_stack = []
+			@m = nil
+			@already = ""
+			self.state = :inside_element
+		end
+
+		attr_accessor :state # = :inside_element, :inside_tag, :inside_comment,
+		
+		def eat_this(line)
+			@rest = line  + @rest
+			things_read = 0
+			until @rest.empty?
+				case self.state
+					when :inside_comment
+						if @m = CommentEnd.match(@rest)
+							@already += @m.pre_match + @m.to_s
+							@rest = @m.post_match
+							self.state = :inside_element
+						else 
+							@already += @rest 
+							@rest = ""
+							self.state = :inside_comment
+						end
+					when :inside_element 
+						if @m = CommentStart.match(@rest)
+							things_read += 1
+							@already += @m.pre_match + @m.to_s
+							@rest = @m.post_match
+							self.state = :inside_comment
+						elsif @m = Tag.match(@rest) then
+							my_debug "#{@state}: Tag: #{@m.to_s.inspect}"
+							things_read += 1
+							handle_tag
+							self.state = :inside_element
+						elsif @m = PartialTag.match(@rest) then
+							my_debug "#{@state}: PartialTag: #{@m.to_s.inspect}"
+							@already += @m.pre_match 
+							@rest = @m.post_match
+							@partial_tag = @m.to_s
+							self.state = :inside_tag
+						elsif @m = EverythingElse.match(@rest)
+							my_debug "#{@state}: Everything: #{@m.to_s.inspect}"
+							@already += @m.pre_match + @m.to_s
+							@rest = @m.post_match
+							self.state = :inside_element
+						else
+							error "Malformed HTML: not complete: #{@rest.inspect}"
+						end
+					when :inside_tag
+						if @m = /^[^>]*>/.match(@rest) then
+							my_debug "#{@state}: inside_tag: matched #{@m.to_s.inspect}"
+							@partial_tag += @m.to_s
+							my_debug "#{@state}: inside_tag: matched TOTAL: #{@partial_tag.to_s.inspect}"
+							@rest = @partial_tag + @m.post_match
+							@partial_tag = nil
+							self.state = :inside_element
+						else
+							@partial_tag += @rest
+							@rest = ""
+							self.state = :inside_tag
+						end
+					else
+						raise "Bug bug: state = #{self.state.inspect}"
+				end # not inside comment
+				
+#				puts inspect
+#				puts "Read: #{@tag_stack.inspect}"
+				break if is_finished? and things_read>0	
+			end
+		end
+
+		def handle_tag()
+			@already += @m.pre_match
+			@rest = @m.post_match
+
+			is_closing = !!@m[1]
+			tag = @m[2]
+			attributes = @m[3].to_s
+		
+			is_single = false
+			if attributes[-1] == ?/ # =~ /\A(.*)\/\Z/
+				attributes = attributes[0, attributes.size-1]
+				is_single = true
+			end
+
+			my_debug "Attributes: #{attributes.inspect}"
+			my_debug "READ TAG #{@m.to_s.inspect} tag = #{tag} closing? #{is_closing} single = #{is_single}"
+	
+			if TO_SANITIZE.include? tag 
+				attributes.strip!
+		#		puts "Attributes: #{attributes.inspect}"
+				if attributes.size > 0
+					@already +=  '<%s %s />' % [tag, attributes]
+				else
+					@already +=  '<%s />' % [tag]
+				end
+			elsif is_closing
+				@already += @m.to_s
+				if @tag_stack.empty?
+					error "Malformed: closing tag #{tag.inspect} "+
+					      "in empty list"
+				end 
+				if @tag_stack.last != tag
+					error "Malformed: tag <#{tag}> "+
+					      "closes <#{@tag_stack.last}>"
+				end
+				@tag_stack.pop
+			else 
+				@already += @m.to_s
+				
+				if not is_single
+					@tag_stack.push(tag) 
+					my_debug "Pushing #{tag.inspect} when read #{@m.to_s.inspect}"
+				end
+			end
+		end
+		def error(s)
+			raise Exception, "Error: #{s} \n"+ inspect, caller
+		end
+
+		def inspect; "HTML READER\n state=#{self.state} "+
+			"match=#{@m.to_s.inspect}\n"+
+			"Tag stack = #{@tag_stack.inspect} \n"+
+			"Before:\n"+
+			add_tabs(@already,1,'|')+"\n"+
+			"After:\n"+
+			add_tabs(@rest,1,'|')+"\n"
+			
+		end
+		
+		
+		def stuff_you_read
+			@already
+		end
+		
+		def rest() @rest end
+		
+		def is_finished?
+			(self.state == :inside_element)  and @tag_stack.empty?
+		end
+	end # html helper 
+
+end end end end