qwik2md 0.1.0

data/vendor/qwik/lib/qwik/parse-plugin.rb

@@ -0,0 +1,249 @@
+# Copyright (C) 2003-2006 Kouichirou Eto, All rights reserved.
+# This is free software with ABSOLUTELY NO WARRANTY.
+# You can redistribute it and/or modify it under the terms of the GNU GPL 2.
+
+$LOAD_PATH.unshift '..' unless $LOAD_PATH.include? '..'
+require 'qwik/parser'
+
+module Qwik
+  class Plugin
+    # Split a string into paragraphs.
+    def self.split(str)
+      paras = []
+
+      cur_str = nil
+      in_plugin = false
+      str.each_line {|line|
+	if ! in_plugin
+	  if /\A\{\{/ =~ line
+	    paras << cur_str if cur_str
+	    cur_str = line
+	    in_plugin = true
+	    if /\}\}$/ =~ line		# Use $ for new line.
+	      paras << cur_str
+	      cur_str = nil
+	      in_plugin = false
+	    end
+	  else
+	    cur_str = '' if cur_str.nil?
+	    cur_str += line
+	  end
+	else
+	  if /\A\}\}$/ =~ line
+	    cur_str += line
+	    paras << cur_str
+	    cur_str = nil
+	    in_plugin = nil
+	  else
+	    cur_str += line
+	  end
+	end
+      }
+
+      if cur_str
+	paras << cur_str
+      end
+
+      return paras
+    end
+
+    def self.rewrite(paras, pluginname, num)
+      cur_num = 1
+      nparas = []
+      paras.each {|para|
+	if /\A\{\{(\w+)/ =~ para && (name = $1.intern) == pluginname
+	  if cur_num == num
+	    tokens = TextTokenizer.tokenize(para)
+	    tree = TextParser.make_tree(tokens)
+	    raise if 1 < tree.length
+	    plugin = tree[0]
+	    raise if plugin[0] != :plugin
+	    new_plugin = yield(plugin)
+	    new_para = encode(new_plugin)
+	    nparas << new_para
+	  else
+	    nparas << para
+	  end
+	  cur_num += 1
+	else
+	  nparas << para
+	end
+      }
+      return nparas
+    end
+
+    def self.encode(plugin)
+      method = plugin[1][:method]
+      param = plugin[1][:param]
+      content = plugin[2]
+      str = "{{#{method}"
+      str << "(#{param})" if param && ! param.empty?
+      if content
+	content.chomp!
+	str << "
+#{content}
+"
+      end
+      str << "}}
+"
+      return str
+    end
+
+    def self.join(paras)
+      str = paras.join
+      return str
+    end
+  end
+end
+
+if $0 == __FILE__
+  require 'qwik/test-common'
+  $test = true
+end
+
+if defined?($test) && $test
+  class TestPlugin < Test::Unit::TestCase
+    include TestSession
+
+    def ok_sp(e, s)
+      ok_eq(e, Qwik::Plugin.split(s))
+    end
+
+    def test_plugin_split
+      ok_sp([], '')
+      ok_sp(['a'], 'a')
+      ok_sp(['a
+b'], 'a
+b')
+      ok_sp(['{{p}}'], '{{p}}')
+      ok_sp(['a
+', '{{p}}'], 'a
+{{p}}')
+      ok_sp(['a
+', '{{p}}
+', 'b'], 'a
+{{p}}
+b')
+      ok_sp(['a
+', '{{p
+}}
+', 'b'], 'a
+{{p
+}}
+b')
+    end
+
+    def ok_re(expected, paras, num)
+      nparas = Qwik::Plugin.rewrite(paras, :p, num) {|plugin|
+	plugin << 'c'
+	plugin
+      }
+      ok_eq(expected, nparas)
+    end
+
+    def test_plugin_rewrite
+      ok_re(["{{p
+c
+}}
+"],
+	    ['{{p}}'], 1)
+      ok_re(['{{p}}'], ['{{p}}'], 2)
+      ok_re(["a
+", "{{p
+c
+}}
+", "b
+"],
+	    ["a
+", "{{p}}
+", "b
+"], 1)
+      ok_re(["a
+", "{{p
+c
+}}
+", "{{p}}
+", "b
+"],
+	    ["a
+", "{{p}}
+", "{{p}}
+", "b
+"], 1)
+      ok_re(["{{p}}
+", "{{p
+c
+}}
+"],
+	    ["{{p}}
+", '{{p}}'], 2)
+      ok_re(["{{p
+c
+}}
+", '{{q}}'],
+	    ["{{p}}
+", '{{q}}'], 1)
+      ok_re(["{{p}}
+", '{{q}}'],
+	    ["{{p}}
+", '{{q}}'], 2)
+    end
+
+    def ok_en(e, s)
+      ok_eq(e, Qwik::Plugin.encode(s))
+    end
+
+    def test_plugin_encode
+      ok_en("{{m}}
+", [:plugin, {:method=>'m'}])
+      ok_en("{{m}}
+", [:plugin, {:method=>'m', :param=>''}])
+      ok_en("{{m(p)}}
+", [:plugin, {:method=>'m', :param=>'p'}])
+      ok_en("{{m(p)
+
+}}
+", [:plugin, {:method=>'m', :param=>'p'}, ''])
+      ok_en("{{m
+c
+}}
+", [:plugin, {:method=>'m'}, 'c'])
+      ok_en("{{m
+c
+}}
+", [:plugin, {:method=>'m'}, "c
+"])
+      ok_en("{{m
+c
+
+}}
+", [:plugin, {:method=>'m'}, "c
+
+"])
+      ok_en("{{m(p)
+c
+}}
+", [:plugin, {:method=>'m', :param=>'p'}, 'c'])
+      ok_en("{{m(p)
+c
+}}
+", [:plugin, {:method=>'m', :param=>'p'}, "c
+"])
+    end
+
+    def test_plugin_join
+      ok_eq("a
+{{p
+c
+}}
+b
+",
+	    Qwik::Plugin.join(["a
+", "{{p
+c
+}}
+", "b
+"]))
+    end
+  end
+end

data/vendor/qwik/lib/qwik/parser-emode.rb

@@ -0,0 +1,116 @@
+# Copyright (C) 2003-2006 Kouichirou Eto, All rights reserved.
+# This is free software with ABSOLUTELY NO WARRANTY.
+# You can redistribute it and/or modify it under the terms of the GNU GPL 2.
+
+$LOAD_PATH.unshift '..' unless $LOAD_PATH.include? '..'
+
+# Special mode, not for general use.
+
+module Qwik
+  class EmodePreProcessor
+    def self.emode?(str)
+      /\A============================================================/ =~ str
+    end
+
+    HMARK = {
+      'h2' => '*',
+      'h3' => '**'
+    }
+
+    def self.preprocess(str)
+      next_line = nil
+      num = 0
+      ar = []
+      str.each {|line|
+	line.chomp!
+	if next_line
+	  if next_line == 'h2'
+	    num += 1
+	  end
+	  if /\A([●■])(.*)\z/ =~ line
+	    ar << HMARK[next_line] + $2
+	  else
+	    if next_line == 'h2'
+	      ar << '*' + num.to_s
+	    else
+	      ar << "===="
+	    end
+	    next_line = add_line(next_line, ar, line)
+	  end
+	  next_line = nil
+	else
+	  next_line = add_line(next_line, ar, line)
+	end
+      }
+      return ar.join("\n")
+    end
+
+    def self.add_line(next_line, ar, line)
+      case line
+      when /\A(=+)\z/		# only =
+	return 'h2'
+      when /\A(------+)/	# only -
+	return 'h3'
+      when /\A([→・])(.*)\z/
+	ar << '-'+$2
+	return next_line
+      when /\A([●■])(.*)\z/
+	ar << '***'+$2
+	return next_line
+      end
+
+      ignore_chars = [?\s, ?-, ?*, ?>, ?|, ?,, ?{, ?}]
+      if ignore_chars.include?(line[0]) || line.empty?
+	ar << line
+	return next_line
+      end
+
+      ar << line+"{{br}}"
+      return next_line
+    end
+  end
+end
+
+if $0 == __FILE__
+  require 'qwik/testunit'
+  $test = true
+end
+
+if defined?($test) && $test
+  class TestEmodePreProcessor < Test::Unit::TestCase
+    def ok(e, str)
+      str = "============================================================\n"+str
+      ok_eq(e, Qwik::EmodePreProcessor.preprocess(str))
+    end
+
+    def test_all
+      c = Qwik::EmodePreProcessor
+
+      # test_emode?
+      eq(false, !!c.emode?(''))
+      eq(true,  !!c.emode?("============================================================"))
+
+      # test_preprocess
+      ok('*t', "●t")
+      ok("*1\n-t", "・t")
+      ok("*1\nt{{br}}", 't')
+      ok("*t\nt1{{br}}\nt2{{br}}", "●t\nt1\nt2")
+      ok("*t\nt1{{br}}\n====\nt2{{br}}",
+	 "●t\nt1\n------------------------------------------------------------\nt2")
+      #ok("*t\n>t1{{br}}\n>t2{{br}}", "●t\n>t1\n>t2") # impossible...
+      ok("*t\n>t1\n>t2", "●t\n>t1\n>t2")
+      ok("*1\n-", '-')
+      ok('*1',
+	 '------------------------------------------------------------')
+
+      # test_sjis_bug
+      ok("*1\nー{{br}}\nt{{br}}", "ー\nt")
+
+      # test_br
+      ok("*1\n{{{\nt{{br}}\n}}}", "{{{\nt\n}}}")
+
+      # test_multiline
+      ok("*1\na{{br}}\n\nb{{br}}", "a\n\nb")
+    end
+  end
+end

data/vendor/qwik/lib/qwik/parser-inline.rb

@@ -0,0 +1,272 @@
+# Copyright (C) 2003-2006 Kouichirou Eto, All rights reserved.
+# This is free software with ABSOLUTELY NO WARRANTY.
+# You can redistribute it and/or modify it under the terms of the GNU GPL 2.
+
+$LOAD_PATH.unshift '..' unless $LOAD_PATH.include? '..'
+
+module Qwik
+  class InlineParser
+    URL = '(?:http|https|ftp|mailto|file):[a-zA-Z0-9;/?:@&=+$,\-_.!~*\'()#%]+'
+
+    INLINE_PATTERNS = {
+      :"'''"=>:strong,
+      :"''"=>:em,
+      :'=='=>:del,
+    }
+
+    def self.make_tree(line_ar)
+      root = []
+      stack = []
+      stack << root
+      in_tag = {}
+
+      line_ar.each {|t|
+	if t.is_a?(Symbol)
+	  if in_tag[t]
+	    in_tag[t] = nil
+	    if stack.last.length == 1	# bad hack
+	      stack.last << ''
+	    end
+	    stack.pop if 1 < stack.length
+	  else
+	    in_tag[t] = true
+	    ar = [INLINE_PATTERNS[t]]
+	    stack.last << ar
+	    stack << ar
+	  end
+
+	else
+	  if t.is_a? Array
+	    elem = t.first
+	    case elem
+	    when :ref
+	      stack.last << parse_ref(t[1])
+	    when :plugin
+	      stack.last << parse_plugin(t)
+	    when :url
+	      stack.last << [:a, {:href=>t[1]}, t[1]]
+	    else
+	      # do nothing
+	    end
+
+	  else
+	    stack.last << t
+	  end
+	end
+      }
+
+      # bad hack
+      if stack.last.length == 1 && stack.last.is_a?(Array) &&
+	  stack.last.first.is_a?(Symbol)
+	stack.last << ''
+      end
+
+      return root
+    end
+
+    def self.parse_plugin(t)
+      method = t[1].to_s
+      param = t[2].to_s
+      data = t[3].to_s
+      return [:br] if method == 'br'	# shortcut
+      return [:plugin, {:method=>method, :param=>param}, data]
+    end
+
+    def self.parse_ref(uri)
+      text, uri = $1, $2 if /\A([^|]*)\|(.+)\z/ =~ uri
+      if text.nil? or text.empty?
+        text = uri
+      end
+
+      case uri
+      when /\A#{URL}/o
+        if /\.(?:jpg|jpeg|png|gif)\z/i =~ uri
+          [:img, {:src=>uri, :alt=>text}]
+        else
+          [:a, {:href=>uri}, text]
+        end
+      when /\A(.+?):(.+)\z/
+        [:plugin, {:method=>'interwiki', :param=>uri}, text]
+      else
+        uri = "#{uri}.html" unless uri.include?('.')
+        [:a, {:href=>uri}, text]
+      end
+    end
+  end
+end
+
+if $0 == __FILE__
+  if ARGV[0] == '-b'
+    $bench = true
+  else
+    require 'qwik/testunit'
+    $test = true
+  end
+end
+
+if defined?($test) && $test
+  require 'qwik/tokenizer-inline'
+
+  class TestInlineParser < Test::Unit::TestCase
+    def ok(e, str)
+      tokens = Qwik::InlineTokenizer.tokenize(str)
+      tree = Qwik::InlineParser.make_tree(tokens)
+      ok_eq(e, tree)
+    end
+
+    def test_all
+      ok([], '')
+      ok(['t'], 't')
+
+      # test em, strong and del
+      ok([[:em, 't']], "''t''")
+      ok([[:strong, 't']], "'''t'''")
+      ok([[:strong, 't', [:em, '']]], "'''t''")
+      ok(["'"], "'")
+      ok(["'", 't'], "'t")
+      ok([[:del, 't']], '==t==')
+      ok(['='], '=')
+      ok(['=', 't'], '=t')
+      ok(['a ',[:em,'b ',[:strong,'c'],' d'],' e'], "a ''b '''c''' d'' e")
+      # FIXME: Take care of incomplete patterns.
+      ok([[:em, 'a', [:strong, 'b', [:del, 'c']]]], "''a'''b==c")
+      ok([[:strong, '']], "''''''")
+      ok(['a', [:em, 'b', [:strong, 'c'], 'd'], 'e'], "a''b'''c''d'''e")
+      ok(['a', [:del, 'b', [:em, 'c'], 'd'], 'e'], "a==b''c==d''e")
+
+      # test_ref
+      ok([[:a, {:href=>'t.html'}, 't']], '[[t]]')
+      ok([[:a, {:href=>'t.html'}, 's']], '[[s|t]]')
+      ok([[:a, {:href=>'http://e.com/'}, 'e']], '[[e|http://e.com/]]')
+      ok([[:plugin, {:method=>'interwiki', :param=>'w:p'}, 'w:p']],
+	 '[[w:p]]')
+      ok([']'], ']')
+      ok(['[', 't', ']'], '[t]')
+      ok([[:a, {:href=>'http://e.com/'}, 't']], '[http://e.com/ t]')
+
+      ok([[:a, {:href=>'C++.html'}, 'C++']], '[[C++]]')
+
+      # test_plugin
+      ok([[:plugin, {:method=>'t', :param=>''}, '']], '{{t}}')
+      ok([[:plugin, {:method=>'t', :param=>'a'}, '']], '{{t(a)}}')
+      ok(['{', 't', '}'], '{t}')
+      #ok(['{', '{', '}}'], '{{}}')	# regexp version.
+      ok(['{{}}'], '{{}}')		# strscan version.
+      ok([[:br]], '{{br}}') # shotcut
+
+      # test_url
+      ok([[:a, {:href=>'http://e.com/'}, 'http://e.com/']], 'http://e.com/')
+      ok([[:a, {:href=>'https://e.com/'}, 'https://e.com/']],
+	 'https://e.com/')
+      ok(['t ', [:a, {:href=>'http://e.com/'}, 'http://e.com/'], ' a'],
+	 't http://e.com/ a')
+
+      # test etc. at a time
+      ok(['a ', [:em, 'b'], ' ', [:strong, 'c'], ' ', [:del, 'd'], ' ',
+	   [:a, {:href=>'e.html'}, 'e'], ' ',
+	   [:plugin, {:method=>'f', :param=>''}, ''], ' ',
+	   [:a, {:href=>'http://e.com/'}, 'http://e.com/'], ' g'],
+	 "a ''b'' '''c''' ==d== [[e]] {{f}} http://e.com/ g")
+
+      # test_img
+      ok([[:img, {:alt=>'http://e.com/t.jpg', :src=>'http://e.com/t.jpg'}]],
+	 '[[http://e.com/t.jpg]]')
+      ok([[:img, {:alt=>'m', :src=>'http://e.com/t.jpg'}]],
+	 '[[m|http://e.com/t.jpg]]')
+
+      # test_security
+      ok([[:a, {:href=>'t.html'}, 't']], '[[t]]')
+      ok([[:a, {:href=>'http://e.com/'}, 'http://e.com/']],
+	 '[[http://e.com/]]')
+      ok([[:plugin, {:param=>'javascript:t', :method=>'interwiki'},
+	     'javascript:t']],
+	 '[[javascript:t]]') # Treated as InterWiki
+      ok([[:a, {:href=>"&{alert('hello');};.html"}, "&{alert('hello');};"]],
+	 "[[&{alert('hello');};]]")
+
+      # test_bug
+      ok([[:a, {:href=>"\226].html"}, "\226]"]], '[[望]]')
+    end
+  end
+
+  class TestInlineParser_parse_ref < Test::Unit::TestCase
+    def ok(e, str)
+      ok_eq(e, Qwik::InlineParser.parse_ref(str))
+    end
+
+    def test_all
+      # normal ref
+      ok([:a, {:href=>'t.html'}, 't'], 't')
+      ok([:a, {:href=>'http://e.com/'}, 'http://e.com/'], 'http://e.com/')
+
+      # with title
+      ok([:a, {:href=>'t.html'}, 's'], 's|t')
+      ok([:a, {:href=>'http://e.com/'}, 's'], 's|http://e.com/')
+
+      # interwiki
+      ok([:plugin, {:param=>'s:t', :method=>'interwiki'}, 's:t'], 's:t')
+      ok([:plugin, {:param=>'s:t', :method=>'interwiki'}, 'foo'], 'foo|s:t')
+
+      # test_plugin_ref
+      ok([:a, {:href=>'.attach'}, '.attach'], '.attach')
+      ok([:a, {:href=>'.attach/t.txt'}, '.attach/t.txt'], '.attach/t.txt')
+      ok([:a, {:href=>'.attach/s t.txt'}, '.attach/s t.txt'],
+	 '.attach/s t.txt')
+
+      # test_img
+      ok([:img, {:alt=>'http://e.com/t.jpg', :src=>'http://e.com/t.jpg'}],
+	 'http://e.com/t.jpg')
+      ok([:img, {:alt=>'s', :src=>'http://e.com/t.jpg'}],
+	 's|http://e.com/t.jpg')
+
+      # test security
+      ok([:a, {:href=>"&{alert('hello');};.html"}, "&{alert('hello');};"],
+	 "&{alert('hello');};")
+
+      # test_sjis_bug
+      ok_eq("\203|\203X", 'ポス')
+      ok([:a, {:href=>"\203|\203X.html"}, "\203|\203X"], 'ポス')
+
+      # test_bug
+      ok_eq("\226]", '望')
+      ok([:a, {:href=>"\226].html"}, "\226]"], '望')
+
+      # abnormal cases
+      ok([:a, {:href=>'|.html'}, '|'],      '|')
+      ok([:a, {:href=>'|.html'}, '|'],      '||')
+      ok([:a, {:href=>'||.html'}, '||'],    '|||')
+      ok([:a, {:href=>'s|.html'}, 's|'],    's|')
+      ok([:a, {:href=>'s.html'},  's'],    '|s')
+      ok([:a, {:href=>'http://example.com'}, 'http://example.com'],    '|http://example.com')
+      ok([:plugin, {:param=>'s:t', :method=>'interwiki'}, 's:t'], '|s:t')
+    end
+  end
+end
+
+if defined?($bench) && $bench
+  require 'qwik/bench-module-session'
+
+  def ok(e, str)
+    tree = Qwik::InlineParser.parse(str)
+  end
+
+  def main
+    n = 10000
+
+# 10000 times.
+#  5.460000   0.220000   5.680000 (  5.683599)
+#  5.460000   0.180000   5.640000 (  5.641708)
+#  5.480000   0.350000   5.830000 (  5.840561)
+
+    BenchmarkModule::benchmark {
+      n.times {
+	ok(['a ', [:em, 'b'], ' ', [:strong, 'c'], ' ', [:del, 'd'], ' ',
+	     [:a, {:href=>'e.html'}, 'e'], ' ',
+	     [:plugin, {:method=>'f', :param=>''}, ''], ' ',
+	     [:a, {:href=>'http://e.com/'}, 'http://e.com/'], ' g'],
+	   "a ''b'' '''c''' ==d== [[e]] {{f}} http://e.com/ g")
+      }
+    }
+  end
+  main
+end