dtext_rb 1.0.0

data/ext/dtext/extconf.rb

@@ -0,0 +1,8 @@
+require "mkmf"
+
+pkg_config "glib-2.0"
+
+have_library "glib-2.0"
+have_header "glib.h"
+have_header "dtext.h"
+create_makefile "dtext/dtext"

data/lib/dtext.rb

@@ -0,0 +1 @@
+require "dtext/dtext"

data/lib/dtext_ruby.rb

@@ -0,0 +1,326 @@
+require 'cgi'
+require 'uri'
+
+class DTextRuby
+  MENTION_REGEXP = /(?<=^| )@\S+/
+  
+  def self.u(string)
+    CGI.escape(string)
+  end
+
+  def self.h(string)
+    CGI.escapeHTML(string)
+  end
+
+  def self.strip_blocks(string, tag)
+    blocks = string.scan(/\[\/?#{tag}\]|.+?(?=\[\/?#{tag}\]|$)/m)
+    n = 0
+    stripped = ""
+    blocks.each do |block|
+      case block
+      when "[#{tag}]"
+        n += 1
+
+      when "[/#{tag}]"
+        n -= 1
+
+      else
+        if n == 0
+          stripped += block
+        end
+      end
+    end
+
+    stripped.strip
+  end
+
+  def self.parse_inline(str, options = {})
+    str.gsub!(/&/, "&amp;")
+    str.gsub!(/</, "&lt;")
+    str.gsub!(/>/, "&gt;")
+    str.gsub!(/\n/m, "<br>") unless options[:ignore_newlines]
+    str.gsub!(/\[b\](.+?)\[\/b\]/i, '<strong>\1</strong>')
+    str.gsub!(/\[i\](.+?)\[\/i\]/i, '<em>\1</em>')
+    str.gsub!(/\[s\](.+?)\[\/s\]/i, '<s>\1</s>')
+    str.gsub!(/\[u\](.+?)\[\/u\]/i, '<u>\1</u>')
+    str.gsub!(/\[tn\](.+?)\[\/tn\]/i, '<p class="tn">\1</p>')
+
+    str = parse_mentions(str)
+    str = parse_links(str)
+    str = parse_aliased_wiki_links(str)
+    str = parse_wiki_links(str)
+    str = parse_post_links(str)
+    str = parse_id_links(str)
+    str
+  end
+
+  def self.parse_mentions(str)
+    str.gsub!(MENTION_REGEXP) do |name| 
+      next name unless name =~ /[a-z0-9]/i
+
+      if name =~ /([:;,.!?\)\]<>])$/
+        name.chop!
+        ch = $1
+      else
+        ch = ""
+      end
+
+      '<a href="/users?name=' + u(CGI.unescapeHTML(name[1..-1])) + '">' + name + '</a>' + ch
+    end
+    str
+  end
+
+  def self.parse_table_elements(str)
+    str = parse_inline(str, :ignore_newlines => true)
+    str.gsub!(/\[(\/?(?:tr|td|th|thead|tbody))\]/, '<\1>')
+    str
+  end
+
+  def self.parse_links(str)
+    str.gsub(/("[^"]+":(https?:\/\/|\/)[^\s\r\n<>]+|https?:\/\/[^\s\r\n<>]+|"[^"]+":\[(https?:\/\/|\/)[^\s\r\n<>\]]+\])+/) do |url|
+      ch = ""
+
+      if url =~ /^"([^"]+)":\[(.+)\]$/
+        text = $1
+        url = $2
+      else
+        if url =~ /^"([^"]+)":(.+)$/
+          text = $1
+          url = $2
+        else
+          text = url
+        end
+
+        if url =~ /([;,.!?\)\]<>])$/
+          url.chop!
+          ch = $1
+        end
+      end
+
+      '<a href="' + url + '">' + text + '</a>' + ch
+    end
+  end
+
+  def self.parse_aliased_wiki_links(str)
+    str.gsub(/\[\[([^\|\]]+)\|([^\]]+)\]\]/m) do
+      text = CGI.unescapeHTML($2)
+      title = CGI.unescapeHTML($1).tr(" ", "_").downcase
+      %{<a href="/wiki_pages/show_or_new?title=#{u(title)}">#{h(text)}</a>}
+    end
+  end
+
+  def self.parse_wiki_links(str)
+    str.gsub(/\[\[([^\]]+)\]\]/) do
+      text = CGI.unescapeHTML($1)
+      title = text.tr(" ", "_").downcase
+      %{<a href="/wiki_pages/show_or_new?title=#{u(title)}">#{h(text)}</a>}
+    end
+  end
+
+  def self.parse_post_links(str)
+    str.gsub(/\{\{([^\}]+)\}\}/) do
+      tags = CGI.unescapeHTML($1)
+      %{<a rel="nofollow" href="/posts?tags=#{u(tags)}">#{h(tags)}</a>}
+    end
+  end
+
+  def self.parse_id_links(str)
+    str = str.gsub(/\bpost #(\d+)/i, %{<a href="/posts/\\1">post #\\1</a>})
+    str = str.gsub(/\bforum #(\d+)/i, %{<a href="/forum_posts/\\1">forum #\\1</a>})
+    str = str.gsub(/\btopic #(\d+)(?!\/p\d|\d)/i, %{<a href="/forum_topics/\\1">topic #\\1</a>})
+    str = str.gsub(/\btopic #(\d+)\/p(\d+)/i, %{<a href="/forum_topics/\\1?page=\\2">topic #\\1/p\\2</a>})
+    str = str.gsub(/\bcomment #(\d+)/i, %{<a href="/comments/\\1">comment #\\1</a>})
+    str = str.gsub(/\bpool #(\d+)/i, %{<a href="/pools/\\1">pool #\\1</a>})
+    str = str.gsub(/\buser #(\d+)/i, %{<a href="/users/\\1">user #\\1</a>})
+    str = str.gsub(/\bartist #(\d+)/i, %{<a href="/artists/\\1">artist #\\1</a>})
+    str = str.gsub(/\bissue #(\d+)/i, %{<a href="https://github.com/r888888888/danbooru/issues/\\1">issue #\\1</a>})
+    str = str.gsub(/\bpixiv #(\d+)(?!\/p\d|\d)/i, %{<a href="http://www.pixiv.net/member_illust.php?mode=medium&illust_id=\\1">pixiv #\\1</a>})
+    str = str.gsub(/\bpixiv #(\d+)\/p(\d+)/i, %{<a href="http://www.pixiv.net/member_illust.php?mode=manga_big&illust_id=\\1&page=\\2">pixiv #\\1/p\\2</a>})
+  end
+
+  def self.parse_list(str, options = {})
+    html = ""
+    current_item = ""
+    layout = []
+    nest = 0
+
+    str.split(/\n/).each do |line|
+      if line =~ /^\s*(\*+) (.+)/
+        if nest > 0
+          html += "<li>#{current_item}</li>"
+        elsif not current_item.strip.empty?
+          html += "<p>#{current_item}</p>"
+        end
+
+        nest = $1.size
+        current_item = parse_inline($2)
+      else
+        current_item += parse_inline(line)
+      end
+
+      if nest > layout.size
+        html += "<ul>"
+        layout << "ul"
+      end
+
+      while nest < layout.size
+        elist = layout.pop
+        if elist
+          html += "</#{elist}>"
+        end
+      end
+    end
+
+    html += "<li>#{current_item}</li>"
+
+    while layout.any?
+      elist = layout.pop
+      html += "</#{elist}>"
+    end
+
+    html
+  end
+
+  def self.parse(str, options = {})
+    return "" if str.nil?
+
+    # Make sure quote tags are surrounded by newlines
+
+    unless options[:inline]
+      str.gsub!(/\s*\[quote\](?!\])\s*/m, "\n\n[quote]\n\n")
+      str.gsub!(/\s*\[\/quote\]\s*/m, "\n\n[/quote]\n\n")
+      str.gsub!(/\s*\[code\](?!\])/m, "\n\n[code]\n\n")
+      str.gsub!(/\[\/code\]\s*/m, "\n\n[/code]\n\n")
+      str.gsub!(/\s*\[spoilers?\](?!\])\s*/m, "\n\n[spoiler]\n\n")
+      str.gsub!(/\s*\[\/spoilers?\]\s*/m, "\n\n[/spoiler]\n\n")
+      str.gsub!(/^(h[1-6]\.\s*.+)$/, "\n\n\\1\n\n")
+      str.gsub!(/\s*\[expand(\=[^\]]*)?\](?!\])\s*/m, "\n\n[expand\\1]\n\n")
+      str.gsub!(/\s*\[\/expand\]\s*/m, "\n\n[/expand]\n\n")
+      str.gsub!(/\s*\[table\](?!\])\s*/m, "\n\n[table]\n\n")
+      str.gsub!(/\s*\[\/table\]\s*/m, "\n\n[/table]\n\n")
+    end
+
+    str.gsub!(/(?:\r?\n){3,}/, "\n\n")
+    str.strip!
+    blocks = str.split(/(?:\r?\n){2}/)
+    stack = []
+    flags = {}
+
+    html = blocks.map do |block|
+      case block
+      when /\A(h[1-6])\.\s*(.+)\Z/
+        tag = $1
+        content = $2
+
+        if options[:inline]
+          "<h6>" + parse_inline(content, options) + "</h6>"
+        else
+          "<#{tag}>" + parse_inline(content, options) + "</#{tag}>"
+        end
+
+      when /^\s*\*+ /
+        parse_list(block, options)
+
+      when "[quote]"
+        if options[:inline]
+          ""
+        else
+          stack << "blockquote"
+          "<blockquote>"
+        end
+
+      when "[/quote]"
+        if options[:inline]
+          ""
+        elsif stack.last == "blockquote"
+          stack.pop
+          '</blockquote>'
+        else
+          ""
+        end
+
+      when "[spoiler]"
+        stack << "spoiler"
+        '<div class="spoiler">'
+
+      when "[/spoiler]"
+        if stack.last == "spoiler"
+          stack.pop
+          "</div>"
+        else
+          ""
+        end
+
+      when "[table]"
+        stack << "table"
+        flags[:table] = true
+        '<table class="striped">'
+
+      when "[/table]"
+        if stack.last == "table"
+          stack.pop
+          flags[:table] = false
+          "</table>"
+        else
+          ""
+        end
+
+      when /\[code\](?!\])/
+        flags[:code] = true
+        stack << "pre"
+        '<pre>'
+
+      when /\[\/code\](?!\])/
+        flags[:code] = false
+        if stack.last == "pre"
+          stack.pop
+          "</pre>"
+        else
+          ""
+        end
+
+      when /\[expand(?:\=([^\]]*))?\](?!\])/
+        stack << "expandable"
+        expand_html = '<div class="expandable"><div class="expandable-header">'
+        expand_html << "<span>#{h($1)}</span>" if $1.present?
+        expand_html << '<input type="button" value="Show" class="expandable-button"/></div>'
+        expand_html << '<div class="expandable-content">'
+        expand_html
+
+      when /\[\/expand\](?!\])/
+        if stack.last == "expandable"
+          stack.pop
+          '</div></div>'
+        end
+
+      else
+        if flags[:code]
+          CGI.escape_html(block) + "\n\n"
+        elsif flags[:table]
+          parse_table_elements(block)
+        else
+          '<p>' + parse_inline(block) + '</p>'
+        end
+      end
+    end
+
+    stack.reverse.each do |tag|
+      if tag == "blockquote"
+        html << "</blockquote>"
+      elsif tag == "div"
+        html << "</div>"
+      elsif tag == "pre"
+        html << "</pre>"
+      elsif tag == "spoiler"
+        html << "</div>"
+      elsif tag == "expandable"
+        html << "</div></div>"
+      elsif tag == "table"
+        html << "</table>"
+      end
+    end
+
+    html.join("")
+  end
+end
+

data/test/dtext_test.rb

@@ -0,0 +1,211 @@
+require 'minitest/autorun'
+require 'dtext/dtext'
+
+class DTextTest < Minitest::Test
+  def assert_parse(expected, input)
+    assert_equal(expected, DTextRagel.parse(input))
+  end
+
+  def test_mentions
+    assert_parse('<p><a rel="nofollow" href="/users?name=bob">@bob</a></p>', "@bob")
+    assert_parse('<p>hi <a rel="nofollow" href="/users?name=bob">@bob</a></p>', "hi @bob")
+    assert_parse('<p>this is not @.@ @_@ <a rel="nofollow" href="/users?name=bob">@bob</a></p>', "this is not @.@ @_@ @bob")
+    assert_parse('<p>multiple <a rel="nofollow" href="/users?name=bob">@bob</a> <a rel="nofollow" href="/users?name=anna">@anna</a></p>', "multiple @bob @anna")
+  end
+
+  def test_sanitize_heart
+    assert_parse('<p>&lt;3</p>', "<3")
+  end
+
+  def test_sanitize_less_than
+    assert_parse('<p>&lt;</p>', "<")
+  end
+
+  def test_sanitize_greater_than
+    assert_parse('<p>&gt;</p>', ">")
+  end
+
+  def test_sanitize_ampersand
+    assert_parse('<p>&amp;</p>', "&")
+  end
+
+  def test_wiki_links
+    assert_parse("<p>a <a href=\"/wiki_pages/show_or_new?title=b\">b</a> c</p>", "a [[b]] c")
+  end
+
+  def test_wiki_links_spoiler
+    assert_parse("<p>a <a href=\"/wiki_pages/show_or_new?title=spoiler\">spoiler</a> c</p>", "a [[spoiler]] c")
+  end
+
+  def test_spoilers_inline
+    assert_parse("<p>this is <span class=\"spoiler\">an inline spoiler</span>.</p>", "this is [spoiler]an inline spoiler[/spoiler].")
+  end
+
+  def test_spoilers_block
+    assert_parse("<p>this is</p><div class=\"spoiler\"><p>a block spoiler</p></div><p>.</p>", "this is\n\n[spoiler]\na block spoiler\n[/spoiler].")
+  end
+
+  def test_spoilers_with_no_closing_tag_1
+    assert_parse("<div class=\"spoiler\"><p>this is a spoiler with no closing tag</p><p>new text</p></div>", "[spoiler]this is a spoiler with no closing tag\n\nnew text")
+  end
+
+  def test_spoilers_with_no_closing_tag_2
+    assert_parse("<div class=\"spoiler\"><p>this is a spoiler with no closing tag<br>new text</p></div>", "[spoiler]this is a spoiler with no closing tag\nnew text")
+  end
+
+  def test_spoilers_with_no_closing_tag_block
+    assert_parse("<div class=\"spoiler\"><p>this is a block spoiler with no closing tag</p></div>", "[spoiler]\nthis is a block spoiler with no closing tag")
+  end
+
+  def test_spoilers_nested
+    assert_parse("<div class=\"spoiler\"><p>this is <span class=\"spoiler\">a nested</span> spoiler</p></div>", "[spoiler]this is [spoiler]a nested[/spoiler] spoiler[/spoiler]")
+  end
+
+  def test_paragraphs
+    assert_parse("<p>abc</p>", "abc")
+  end
+
+  def test_paragraphs_with_newlines_1
+    assert_parse("<p>a<br>b<br>c</p>", "a\nb\nc")
+  end
+
+  def test_paragraphs_with_newlines_2
+    assert_parse("<p>a</p><p>b</p>", "a\n\nb")
+  end
+
+  def test_headers
+    assert_parse("<h1>header</h1>", "h1. header")
+  end
+
+  def test_quote_blocks
+    assert_parse('<blockquote><p>test</p></blockquote>', "[quote]\ntest\n[/quote]")
+  end
+
+  def test_quote_blocks_nested
+    assert_parse("<blockquote><p>a</p><blockquote><p>b</p></blockquote><p>c</p></blockquote>", "[quote]\na\n[quote]\nb\n[/quote]\nc\n[/quote]")
+  end
+
+  def test_quote_blocks_nested_spoiler
+    assert_parse("<blockquote><p>a<br><span class=\"spoiler\">blah</span><br>c</p></blockquote>", "[quote]\na\n[spoiler]blah[/spoiler]\nc[/quote]")
+    assert_parse("<blockquote><p>a</p><div class=\"spoiler\"><p>blah</p></div><p>c</p></blockquote>", "[quote]\na\n\n[spoiler]blah[/spoiler]\n\nc[/quote]")
+  end
+
+  def test_quote_blocks_nested_expand
+    assert_parse("<blockquote><p>a</p><div class=\"expandable\"><div class=\"expandable-header\"><input type=\"button\" value=\"Show\" class=\"expandable-button\"/></div><div class=\"expandable-content\"><p>b</p></div></div><p>c</p></blockquote>", "[quote]\na\n[expand]\nb\n[/expand]\nc\n[/quote]")
+  end
+
+  def test_code
+    assert_parse("<pre>for (i=0; i&lt;5; ++i) {\n  printf(1);\n}\n\nexit(1);</pre>", "[code]for (i=0; i<5; ++i) {\n  printf(1);\n}\n\nexit(1);")
+  end
+
+  def test_urls
+    assert_parse('<p>a <a href="http://test.com">http://test.com</a> b</p>', p('a http://test.com b'))
+  end
+
+  def test_urls_with_newline
+    assert_parse('<p><a href="http://test.com">http://test.com</a><br>b</p>', "http://test.com\nb")
+  end
+
+  def test_urls_with_paths
+    assert_parse('<p>a <a href="http://test.com/~bob/image.jpg">http://test.com/~bob/image.jpg</a> b</p>', p('a http://test.com/~bob/image.jpg b'))
+  end
+
+  def test_urls_with_fragment
+    assert_parse('<p>a <a href="http://test.com/home.html#toc">http://test.com/home.html#toc</a> b</p>', p('a http://test.com/home.html#toc b'))
+  end
+
+  def test_auto_urls
+    assert_parse('<p>a <a href="http://test.com">http://test.com</a>. b</p>', p('a http://test.com. b'))
+  end
+
+  def test_auto_urls_in_parentheses
+    assert_parse('<p>a (<a href="http://test.com">http://test.com</a>) b</p>', p('a (http://test.com) b'))
+  end
+
+  def test_old_style_links
+    assert_parse('<p><a href="http://test.com">test</a></p>', p('"test":http://test.com'))
+  end
+
+  def test_old_style_links_with_special_entities
+    assert_parse('<p>"1" <a href="http://three.com">2 &amp; 3</a></p>', p('"1" "2 & 3":http://three.com'))
+  end
+
+  def test_new_style_links
+    assert_parse('<p><a href="http://test.com">test</a></p>', p('"test":[http://test.com]'))
+  end
+
+  def test_new_style_links_with_parentheses
+    assert_parse('<p><a href="http://test.com/(parentheses)">test</a></p>', p('"test":[http://test.com/(parentheses)]'))
+    assert_parse('<p>(<a href="http://test.com/(parentheses)">test</a>)</p>', p('("test":[http://test.com/(parentheses)])'))
+    assert_parse('<p>[<a href="http://test.com/(parentheses)">test</a>]</p>', p('["test":[http://test.com/(parentheses)]]'))
+  end
+
+  def test_lists_1
+    assert_parse('<ul><li>a</li></ul>', p('* a'))
+  end
+
+  def test_lists_2
+    assert_parse('<ul><li>a</li><li>b</li></ul>', "* a\n* b")
+  end
+
+  def test_lists_nested
+    assert_parse('<ul><li>a</li><ul><li>b</li></ul></ul>', "* a\n** b")
+  end
+
+  def test_lists_inline
+    assert_parse('<ul><li><a href="/posts/1">post #1</a></li></ul>', "* post #1")
+  end
+
+  def test_lists_not_preceded_by_newline
+    assert_parse('<p>a<br>b</p><ul><li>c</li><li>d</li></ul>', "a\nb\n* c\n* d")
+  end
+
+  def test_lists_with_multiline_items
+    assert_parse('<p>a</p><ul><li>b<br>c</li><li>d<br>e</li></ul><p>another one</p>', "a\n* b\nc\n* d\ne\n\nanother one")
+    assert_parse('<p>a</p><ul><li>b<br>c</li><ul><li>d<br>e</li></ul></ul><p>another one</p>', "a\n* b\nc\n** d\ne\n\nanother one")
+  end
+
+  def test_inline_tags
+    assert_parse('<p><a rel="nofollow" href="/posts?tags=tag">tag</a></p>', "{{tag}}")
+  end
+
+  def test_inline_tags_conjunction
+    assert_parse('<p><a rel="nofollow" href="/posts?tags=tag1%20tag2">tag1 tag2</a></p>', "{{tag1 tag2}}")
+  end
+
+  def test_inline_tags_special_entities
+    assert_parse('<p><a rel="nofollow" href="/posts?tags=%3C3">&lt;3</a></p>', "{{<3}}")
+  end
+
+  def test_extra_newlines
+    assert_parse('<p>a</p><p>b</p>', "a\n\n\n\n\n\n\nb\n\n\n\n")
+  end
+
+  def test_complex_links_1
+    assert_parse("<p><a href=\"/wiki_pages/show_or_new?title=1\">2 3</a> | <a href=\"/wiki_pages/show_or_new?title=4\">5 6</a></p>", "[[1|2 3]] | [[4|5 6]]")
+  end
+
+  def test_complex_links_2
+    assert_parse("<p>Tags <strong>(<a href=\"/wiki_pages/show_or_new?title=howto:tag\">Tagging Guidelines</a> | <a href=\"/wiki_pages/show_or_new?title=howto:tag_checklist\">Tag Checklist</a> | <a href=\"/wiki_pages/show_or_new?title=tag_groups\">Tag Groups</a>)</strong></p>", "Tags [b]([[howto:tag|Tagging Guidelines]] | [[howto:tag_checklist|Tag Checklist]] | [[Tag Groups]])[/b]")
+  end
+
+  def test_table
+    assert_parse("<table class=\"striped\"><thead><tr><th>header</th></tr></thead><tbody><tr><td><a href=\"/posts/100\">post #100</a></td></tr></tbody></table>", "[table][thead][tr][th]header[/th][/tr][/thead][tbody][tr][td]post #100[/td][/tr][/tbody][/table]")
+  end
+
+  def test_table_with_newlines
+    assert_parse("<table class=\"striped\"><thead><tr><th>header</th></tr></thead><tbody><tr><td><a href=\"/posts/100\">post #100</a></td></tr></tbody></table>", "[table]\n[thead]\n[tr]\n[th]header[/th][/tr][/thead][tbody][tr][td]post #100[/td][/tr][/tbody][/table]")
+  end
+
+  def test_forum_links
+    assert_parse('<p><a href="/forum_topics/1234?page=4">topic #1234/p4</a></p>', "topic #1234/p4")
+  end
+
+  def test_boundary_exploit
+    assert_parse('<p><a rel="nofollow" href="/users?name=mack">@mack</a>&lt;</p>', "@mack<")
+  end
+
+  def test_inline_mode
+    assert_equal("hello", DTextRagel.parse("hello", :inline => true).strip)
+  end
+end