qwik2md 0.1.0

data/vendor/qwik/lib/qwik/tokenizer-inline.rb

@@ -0,0 +1,288 @@
+# Copyright (C) 2003-2006 Kouichirou Eto, All rights reserved.
+# This is free software with ABSOLUTELY NO WARRANTY.
+# You can redistribute it and/or modify it under the terms of the GNU GPL 2.
+
+require 'strscan'
+
+$LOAD_PATH.unshift '..' unless $LOAD_PATH.include? '..'
+
+module Qwik
+  class InlineTokenizer
+    URL = '(?:http|https|ftp|mailto|file):[a-zA-Z0-9;/?:@&=+$,\-_.!~*\'()#%]+'
+    SPECIAL = '^\[\]\'=\{\}'
+
+    # Parse a line into tokens using regexp.
+    def self.regexp_tokenize(str)
+      line_ar = []
+
+      while 0 < str.length
+	first_character = str[0]
+	rest = str[1, str.length-1]
+
+	case first_character
+
+	when ?'		# ''t'' or '''t'''
+	  if /\A'''(?!:')/ =~ str
+	    line_ar << :"'''"
+	    str = $'
+	  elsif /\A''/ =~ str
+	    line_ar << :"''"
+	    str = $'
+	  else
+	    line_ar << first_character.chr
+	    str = rest
+	  end
+
+	when ?=		# ==t==
+	  if /\A==/ =~ str
+	    line_ar << :'=='
+	    str = $'
+	  else
+	    line_ar << first_character.chr
+	    str = rest
+	  end
+
+	when ?[		# [[t]]
+
+	  if /\A\[\[(.+?)\]\]/ =~ str	# [[title|url]] format
+	    line_ar << [:ref, $1]
+	    str = $'
+
+	  elsif /\A\[([^\[\]\s]+?) ([^\[\]\s]+?)\]/ =~ str
+	    # [url title] format
+	    line_ar << [:ref, $2+'|'+$1]
+	    str = $'
+
+	  else
+	    line_ar << first_character.chr
+	    str = rest
+	  end
+
+	when ?{		# {{t}}
+	  if /\A\{\{([^\(\)]+?)(?:\((.*?)\))?\s*\}\}/ =~ str # {{t(a)}}
+	    ar = [:plugin, $1]
+	    ar << $2 if $2
+	    line_ar << ar
+	    str = $'
+	  else
+	    line_ar << first_character.chr
+	    str = rest
+	  end
+
+	else
+	  if /\A#{URL}/ =~ str
+	    href = $&
+	    line_ar << [:url, href]
+	    str  = $'
+
+	  elsif /\A[^#{SPECIAL}]+/ =~ str
+	    m = $&
+	    after = $'
+
+	    if /([^a-zA-Z\d]+)(#{URL})/ =~ m
+	      s = $` + $1
+	      line_ar << s
+	      str = $2 + $' + after
+	    else
+	      line_ar << m
+	      str = after
+	    end
+
+	  else
+	    if /\A(.+?)([^#{SPECIAL}])/ =~ str
+	      line_ar << $1
+	      str = $2 + $'
+	    else
+	      line_ar << str
+	      str = ''
+	    end
+	  end
+
+	end
+      end
+
+      return line_ar
+    end
+
+    # Parse a line into tokens using strscan.
+   #def self.strscan_tokenize(str)
+    def self.tokenize(str)
+      line_ar = []
+
+      s = StringScanner.new(str)
+
+      while ! s.eos?
+	if s.scan(/'''(?!:')/)
+	  line_ar << :"'''"
+
+	elsif s.scan(/''(?!:')/)
+	  line_ar << :"''"
+
+	elsif s.scan(/==/)
+	  line_ar << :'=='
+
+	elsif s.scan(/\[\[(.+?)\]\]/)	# [[title|url]] format
+	  line_ar << [:ref, s[1]]
+
+	elsif s.scan(/\[([^\[\]\s]+?) ([^\[\]\s]+?)\]/) # [url title] format
+	  line_ar << [:ref, s[2]+'|'+s[1]]
+
+	elsif s.scan(/\{\{([^\(\)]+?)(?:\((.*?)\))?\s*\}\}/)	# {{t(a)}}
+	  ar = [:plugin, s[1]]
+	  ar << s[2] if s[2]
+	  line_ar << ar
+
+	elsif s.scan(/#{URL}/)
+	  href = s.matched
+	  line_ar << [:url, href]
+
+	elsif s.scan(/[^#{SPECIAL}]+/)
+	  m = s.matched
+
+	  if /([^a-zA-Z\d]+)(#{URL})/ =~ m
+	    ss = $` + $1
+
+	    line_ar << ss
+
+	    skip_str = ss
+	    s.unscan
+	    s.pos = s.pos + skip_str.bytesize
+
+	  else
+	    line_ar << m
+	  end
+
+	elsif s.scan(/(.+?)([^#{SPECIAL}])/)
+	  ss = s[1]
+	  line_ar << ss
+	  s.unscan
+	  s.pos = s.pos + ss.length
+
+	else
+	  ss = s.string
+	  line_ar << ss[s.pos..ss.length]
+	  s.terminate
+
+	end
+      end
+
+      line_ar
+    end
+
+  end
+end
+
+if $0 == __FILE__
+  if ARGV[0] == '-b'
+    $bench = true
+  else
+    $test = true
+  end
+end
+
+if defined?($test) && $test
+  require 'qwik/testunit'
+
+  class TestInlineTokenizer < Test::Unit::TestCase
+    def ok(e, str)
+      ok_eq(e, Qwik::InlineTokenizer.tokenize(str))
+    end
+
+    def test_common
+      ok([], '')
+      ok(['t'], 't')
+
+      # test em, strong and del
+      ok([:"''", 't', :"''"], "''t''")
+      ok([:"'''", 't', :"'''"], "'''t'''")
+      ok(["'"], "'")
+      ok(["'", 't'], "'t")
+      ok([:'==', 't', :'=='], '==t==')
+      ok(['='], '=')
+      ok(['=', 't'], '=t')
+
+      # reference
+      ok([[:ref, 't']], '[[t]]')
+      ok([[:ref, 's|t']], '[[s|t]]')
+      ok([']'], ']')
+      ok(['[', 't', ']'], '[t]')
+      ok([[:ref, 'C++']], '[[C++]]')
+
+      # plugin
+      ok([[:plugin, 't']], '{{t}}')
+      ok([[:plugin, 't', 'a']], '{{t(a)}}')
+      ok(['{', 't', '}'], '{t}')
+
+      # url
+      ok([[:url, 'http://e.com/']], 'http://e.com/')
+      ok(['t ', [:url, 'http://e.com/'], ' a'], 't http://e.com/ a')
+      ok([[:url, 'https://e.com/']], 'https://e.com/')
+
+      # test_all
+      ok(['a ', :"''", 'b', :"''", ' ', :"'''", 'c', :"'''", ' ',
+	       :==, 'd', :==, ' ', [:ref, 'e'], ' ',
+	       [:plugin, 'f'], ' ', [:url, 'http://e.com/'], ' g'],
+	     "a ''b'' '''c''' ==d== [[e]] {{f}} http://e.com/ g")
+
+      # test_wiliki_style
+      ok(['[', 'u', ']'], '[u]')
+      ok([[:ref, 't|u']], '[u t]')
+      ok([[:ref, 't|http://e.com/']], '[http://e.com/ t]')
+
+      # test_ref
+      ok([[:ref, 't|u']], '[[t|u]]')
+      ok([[:ref, 't']], '[[t]]')
+      ok([[:ref, '.attach']], '[[.attach]]')
+      ok([[:ref, '.attach/t.txt']], '[[.attach/t.txt]]')
+      ok([[:ref, '.attach/s t.txt']], '[[.attach/s t.txt]]')
+
+      ok([[:ref, "\203|\203X\203^\201["]], '[[ポスター]]')
+      ok(["\202\240", [:ref, "\203|\203X\203^\201["], "\202\240"],
+	     'あ[[ポスター]]あ')
+
+      # test_bug
+      ok([:"'''", 't', :"''"], "'''t''")
+
+      # test_sjis_bug
+      ok_eq("\226]", '望')
+      ok(["\226]"], '望')
+      ok([[:ref, "\226]"]], '[[望]]')
+    end
+
+  end
+end
+
+if defined?($bench) && $bench
+  require 'qwik/bench-module-session'
+
+  def ok(e, str)
+    tokens = Qwik::InlineTokenizer.tokenize(str)
+    #ok_eq(e, tokens)
+  end
+
+# 10000 times.
+
+# Regexp version.
+#  4.550000   0.990000   5.540000 (  5.527012)
+#  4.600000   0.870000   5.470000 (  5.485405)
+#  4.310000   1.210000   5.520000 (  5.571910)
+
+# strscan version.
+#  4.010000   0.300000   4.310000 (  4.314455)
+#  3.960000   0.340000   4.300000 (  4.305931)
+#  4.080000   0.230000   4.310000 (  4.310803)
+
+  def main
+    n = 10000
+
+    BenchmarkModule::benchmark {
+      n.times {
+	ok(['a ', :"''", 'b', :"''", ' ', :"'''", 'c', :"'''", ' ',
+	     :==, 'd', :==, ' ', [:ref, 'e'], ' ',
+	     [:plugin, 'f'], ' ', [:url, 'http://e.com/'], ' g'],
+	   "a ''b'' '''c''' ==d== [[e]] {{f}} http://e.com/ g")
+      }
+    }
+  end
+  main
+end

data/vendor/qwik/lib/qwik/tokenizer.rb

@@ -0,0 +1,331 @@
+# Copyright (C) 2003-2006 Kouichirou Eto, All rights reserved.
+# This is free software with ABSOLUTELY NO WARRANTY.
+# You can redistribute it and/or modify it under the terms of the GNU GPL 2.
+
+require 'strscan'
+
+module Qwik
+  class TextTokenizer
+    MULTILINE = {
+      :plugin => [
+	/\A\{\{([^\(\)\{\}]+?)(?:\(([^\(\)\{\}]*?)\))?\z/,
+	/\A\}\}\z/
+      ],
+      :pre => [
+	/\A\{\{\{\z/,
+	/\A\}\}\}\z/
+      ],
+      :html => [
+	/\A\<html\>\z/,
+	/\A\<\/html\>\z/
+      ],
+    }
+
+    # Tokenize a text.
+    def self.tokenize(str, br_mode=false)
+      tokens = []
+      in_multiline = {}
+
+      scanner = StringScanner.new(str)
+      while ! scanner.eos?
+	line = scanner.scan(/.*$/)
+	newline = scanner.scan(/\n/)
+
+	line.chomp!
+
+	# At the first, check if it is in a multiline block.
+	last_token = tokens.last
+	if last_token
+	  last_tag = last_token[0]
+	  if in_multiline[last_tag]
+	    if MULTILINE[last_tag][1] =~ line
+	      in_multiline[last_tag] = nil
+	    else
+	      last_token[-1] += "#{line.chomp}\n"
+	    end
+	    next
+	  end
+	end
+
+	line.chomp!
+
+	# preprocess
+	#line.gsub!(/&my-([0-9]+);/) {|m| "{{my_char(#{$1})}}" }
+
+	case line
+	when MULTILINE[:plugin][0]
+	  in_multiline[:plugin] = true
+	  tokens << [:plugin, $1.to_s, $2.to_s, '']
+	when MULTILINE[:pre][0]
+	  in_multiline[:pre] = true
+	  tokens << [:pre, '']
+	when MULTILINE[:html][0]
+	  tokens << [:html, '']
+	  in_multiline[:html] = true
+	when /\A====+\z/, '-- ', /\A----+\z/		# hr
+	  tokens << [:hr]
+	when /\A(\-{1,3})(.+)\z/			# ul
+	  tokens << [:ul, $1.size, $2.to_s.strip]
+	when /\A(\+{1,3})(.+)\z/			# ol
+	  tokens << [:ol, $1.size, $2.to_s.strip]
+	when /\A>(.*)\z/				# blockquote
+	  tokens << [:blockquote, $1.strip]
+	when /\A[ \t](.*)\z/				# pre
+	  tokens << [:pre, $1]
+	when /\A\{\{([^\(\)\{\}]+?)(?:\(([^\(\)\{\}]*?)\))?\}\}\z/	# plugin
+	  tokens << [:plugin, $1.to_s, $2.to_s]
+	when '', /\A#/					# '', or comment.
+	  tokens << [:empty]				# empty line
+	when /\A([,|])(.*)\z/				# pre
+	  re = Regexp.new(Regexp.quote($1), nil)
+	  ar = [:table] + $2.split(re).map {|a| a.to_s }
+	  tokens << ar
+	when /\A:(.*)\z/				# dl
+	  rest = $1
+	  dt, dd = rest.split(':', 2)
+	  if dt && dt.include?('(')
+	    if /\A(.*?\(.*\)[^:]*):(.*)\z/ =~ rest	# FIXME: Bad hack.
+	      dt, dd = $1, $2
+	    end
+	  end
+	  ar = [:dl]
+	  ar << ((dt && ! dt.empty?) ? dt.to_s.strip : nil)
+	  ar << ((dd && ! dd.empty?) ? dd.to_s.strip : nil)
+	  tokens << ar
+	when /\A([*!]{1,5})\s*(.+)\s*\z/		# h
+	  h = $1
+	  s = $2
+	  s = s.strip
+	  if s.empty?		# '* '
+	    inline(tokens, line, br_mode)
+	  else
+	    tokens << [("h#{h.size+1}").intern, s]
+	  end
+	else
+	  inline(tokens, line, br_mode)
+	end
+      end
+
+      return tokens
+    end
+
+    private
+
+    def self.inline(tokens, line, br_mode)
+      if /~\z/ =~ line
+	line = line.sub(/~\z/, '{{br}}')	# //s means shift_jis
+      elsif br_mode
+	line = "#{line}{{br}}"
+      end
+      tokens << [:text, line]
+    end
+  end
+end
+
+if $0 == __FILE__
+  $LOAD_PATH.unshift '..' unless $LOAD_PATH.include? '..'
+  if ARGV[0] == '-b'
+    $bench = true
+  else
+    require 'qwik/testunit'
+    $test = true
+  end
+end
+
+if defined?($test) && $test
+  class TestTokenizer < Test::Unit::TestCase
+    def ok(e, str)
+      ok_eq(e, Qwik::TextTokenizer.tokenize(str))
+    end
+
+    def test_all
+      ok([], '')
+      ok([[:empty]], '#t')
+      ok([[:hr]], '====')
+      ok([[:text, '===']], '===')
+
+      # test_h
+      ok([[:text, '*']], '*')
+      ok([[:h2, 't']], '*t')
+      ok([[:h2, 't']], '* t')
+      ok([[:h2, 't']], '*t ')
+      ok([[:h2, 's t']], '* s t')
+      ok([[:text, '* ']], '* ')
+      ok([[:text, '*']], '*')
+      ok([[:h3, 't']], '**t')
+      ok([[:h4, 't']], '***t')
+      ok([[:h5, 't']], '****t')
+      ok([[:h6, 't']], '*****t')
+
+      # test_!
+      ok([[:h2, 't']], '! t')
+
+      ok([[:text, '-']], '-')
+      ok([[:ul, 1, 't']], '-t')
+      ok([[:ul, 1, 't']], '- t')
+      ok([[:ul, 2, 't']], '--t')
+      ok([[:ul, 3, 't']], '---t')
+      ok([[:ul, 3, '-t']], '----t') # uum...
+      ok([[:ul, 1, '-']], '--')
+      ok([[:hr]], '-- ')
+      ok([[:hr]], '----')
+      ok([[:text, '+']], '+')
+      ok([[:ol, 1, 't']], '+t')
+      ok([[:blockquote, 't']], '>t')
+      ok([[:blockquote, 't']], '> t')
+
+      # test dl
+      ok([[:dl, 'dt', 'dd']], ':dt:dd')
+      ok([[:dl, 'dt', nil]], ':dt')
+      ok([[:dl, nil, nil]], ':')
+      ok([[:dl, nil, 'dd']], '::dd')
+      ok([[:dl, nil, nil]], '::')
+      ok([[:dl, 'dt', 'dd'], [:dl, 'dt2', 'dd2']], ":dt:dd\n:dt2:dd2")
+
+      ok([[:pre, 't']], ' t')
+      ok([[:pre, 't']], "\tt")
+
+      # test_table
+      ok([[:table, 't']], ',t')
+      #ok([[:table, 't']], ', t')
+      ok([[:table, ' t']], ', t')
+      ok([[:table, 's', 't']], ',s,t')
+      ok([[:table, '', 't']], ',,t')
+      ok([[:table, 't']], '|t')
+      ok([[:table, '$t']], ',$t')
+      ok([[:table, '$1']], ',$1')
+
+      ok([[:text, '{t}']], '{t}')
+      ok([[:plugin, 't', '']], '{{t}}')
+      ok([[:plugin, 't', 'a']], '{{t(a)}}')
+      ok([[:plugin, 't', '', '']], '{{t')
+      ok([[:plugin, 't', 'a', '']], '{{t(a)')
+      ok([[:plugin, 't', '', "s\n"]], "{{t\ns\n}}")
+      ok([[:plugin, 't', 'a', "s\n"]], "{{t(a)\ns\n}}")
+
+      ok([[:pre, "s\n"]], "{{{\ns\n}}}")
+      ok([[:pre, "s\nt\n"]], "{{{\ns\nt\n}}}")
+      ok([[:pre, "#s\n"]], "{{{\n#s\n}}}")
+
+      ok([[:empty]], "\n")
+      ok([[:text, 't']], 't')
+      ok([[:text, 's'], [:text, 't']], "s\nt")
+      ok([[:text, 's'], [:empty], [:text, 't']], "s\n\nt")
+
+      # test_html
+      ok([[:text, '<t']], '<t')
+      ok([[:html, '']], '<html>')
+      ok([[:html, "t\n"]], "<html>\nt\n</html>")
+      ok([[:html, "<p>\nt\n</p>\n"]], "<html>\n<p>\nt\n</p>\n</html>")
+
+      # test_sjis_bug
+      ok([[:table, '怖', '怖']], ',怖,怖')
+      ok([[:table, 's', 't']], '|s|t')
+      ok([[:table, '怖', '怖']], '|怖|怖')
+
+      # test_multiline
+      ok([[:text, 's'], [:text, 't']], "s\nt")
+      ok([[:text, 's{{br}}'], [:text, 't{{br}}']], "s~\nt~")
+      ok([[:text, 's'], [:empty], [:text, 't']], "s\n\nt")
+      ok([[:text, "s{{br}}"], [:text, "{{br}}"], [:text, "t{{br}}"]],
+	 "s~\n~\nt~")
+
+      # test_ul
+      ok([[:ul, 1, 't'], [:ul, 1, 't']], "-t\n-t")
+      ok([[:ul, 1, 't'], [:empty], [:ul, 1, 't']], "-t\n\n-t")
+      ok([[:empty], [:ul, 1, 't'], [:ul, 1, 't']], "\n-t\n-t")
+
+      # test_multiline_pre
+      ok([[:pre, 't']], ' t')
+      ok([[:pre, 't1'], [:pre, 't2']], " t1\n t2")
+      ok([[:pre, 't1'], [:empty], [:pre, 't2']], " t1\n\n t2")
+
+      # test_ref
+      # do not parse inline for this moment.
+      ok([[:text, "[[t]]"]], "[[t]]")
+
+      # test_super_pre
+      ok([[:pre, "p\n"], [:text, 't']], "{{{\np\n}}}\nt\n")
+      ok([[:pre, "#p\n"], [:text, 't']], "{{{\n#p\n}}}\nt\n")
+
+      # test_in_bracket # bad hack
+      ok([[:dl, 'dt', 'dd']], ':dt:dd')
+      ok([[:dl, "dt(h:m)dt2", 'dd']], ":dt(h:m)dt2:dd")
+      ok([[:dl, 'u', 'h://e.com/']], ':u:h://e.com/')
+      ok([[:dl, "dt(h:m)", 'h://e.com/']], ":dt(h:m):h://e.com/")
+      ok([[:dl, "dt(h:m)dt2", 'h://e.com/']], ":dt(h:m)dt2:h://e.com/")
+    end
+  end
+
+  class TestTokenizerWithBR < Test::Unit::TestCase
+    def ok(e, str)
+      ok_eq(e, Qwik::TextTokenizer.tokenize(str, true))
+    end
+
+    def test_all
+      ok([[:text, "==={{br}}"]], "===")
+      ok([[:text, "*{{br}}"]], '*')
+      ok([[:text, "* {{br}}"]], '* ')
+      ok([[:text, "*{{br}}"]], '*')
+      ok([[:text, "-{{br}}"]], '-')
+      ok([[:text, "+{{br}}"]], "+")
+      ok([[:text, "{t}{{br}}"]], "{t}")
+      ok([[:text, "t{{br}}"]], 't')
+      ok([[:text, "s{{br}}"], [:text, "t{{br}}"]], "s\nt")
+      ok([[:text, "s{{br}}"], [:empty], [:text, "t{{br}}"]], "s\n\nt")
+
+      # test table
+      ok([[:table, 'a', 'b']], '|a|b|')
+
+      # test_multiline
+      ok([[:text, "s{{br}}"], [:text, "t{{br}}"]], "s\nt")
+      ok([[:text, "s{{br}}"], [:text, "t{{br}}"]], "s~\nt~") # no change
+      ok([[:text, "s{{br}}"], [:empty], [:text, "t{{br}}"]], "s\n\nt")
+      ok([[:text, "s{{br}}"], [:text, "{{br}}"], [:text, "t{{br}}"]],
+	 "s~\n~\nt~") # no change
+
+      # test_ref
+      ok([[:text, "[[t]]{{br}}"]], "[[t]]")
+    end
+  end
+end
+
+if defined?($bench) && $bench
+  require 'qwik/bench-module-session'
+
+  def ok(e, str)
+    tokens = Qwik::TextTokenizer.tokenize(str)
+    #ok_eq(e, tokens)
+  end
+
+  def generate_large_table(table_line_num)
+    str = ''
+    table_line_num.times {|n|
+      str << "|#{n}|1|2|3|4|5|6|7|8|9|0\n"
+    }
+    return str
+  end
+
+# n=10, line=10000
+
+# str.each vesrion.
+#  6.050000   0.660000   6.710000 (  6.701044)
+#  6.060000   0.640000   6.700000 (  6.707144)
+#  6.140000   0.580000   6.720000 (  6.725063)
+
+# strscan version.
+#  5.770000   0.790000   6.560000 (  6.559884)
+#  5.810000   0.750000   6.560000 (  6.568743)
+#  6.030000   0.550000   6.580000 (  6.562864)
+
+  def main
+    n = 10
+    str = generate_large_table(10000)
+    BenchmarkModule::benchmark {
+      n.times {
+	ok([], str)
+      }
+    }
+  end
+  main
+end