wikitext 0.1

data/ext/wikitext_ragel.h

@@ -0,0 +1,17 @@
+// Copyright 2008 Wincent Colaiuta
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#include "token.h"
+
+void next_token(token_t *out, token_t *last_token, char *p, char *pe);

data/spec/autolinking_spec.rb

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+# Copyright 2007-2008 Wincent Colaiuta
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require File.join(File.dirname(__FILE__), 'spec_helper.rb')
+require 'wikitext'
+
+describe Wikitext::Parser, 'autolinking' do
+  before do
+    @parser = Wikitext::Parser.new
+  end
+
+  it 'should default to autolinking on' do
+    @parser.autolink.should == true
+  end
+
+  describe 'on' do
+    it 'should convert HTTP URIs into hyperlinks' do
+      uri = 'http://example.com/'
+      @parser.parse(uri).should == %Q{<p><a href="http://example.com/" class="external">http://example.com/</a></p>\n}
+    end
+
+    it 'should convert HTTPS URIs into hyperlinks' do
+      uri = 'https://example.com/'
+      @parser.parse(uri).should == %Q{<p><a href="https://example.com/" class="external">https://example.com/</a></p>\n}
+    end
+
+    it 'should convert FTP URIs into hyperlinks' do
+      uri = 'ftp://example.com/'
+      @parser.parse(uri).should == %Q{<p><a href="ftp://example.com/" class="external">ftp://example.com/</a></p>\n}
+    end
+
+    it 'should convert mailto URIs into hyperlinks' do
+      uri = 'mailto:user@example.com'
+      @parser.parse(uri).should == %Q{<p><a href="mailto:user@example.com" class="external">mailto:user@example.com</a></p>\n}
+    end
+
+    it 'should convert SVN URIs into hyperlinks' do
+      uri = 'svn://example.com/'
+      @parser.parse(uri).should == %Q{<p><a href="svn://example.com/" class="external">svn://example.com/</a></p>\n}
+    end
+
+    it 'should apple the external_link_class CSS class if set' do
+      uri = 'http://example.com/'
+      @parser.external_link_class = 'bar'
+      @parser.parse(uri).should == %Q{<p><a href="http://example.com/" class="bar">http://example.com/</a></p>\n}
+    end
+
+    it 'should apply no CSS if external_link_class is set to nil' do
+      uri = 'http://example.com/'
+      @parser.external_link_class = nil
+      @parser.parse(uri).should == %Q{<p><a href="http://example.com/">http://example.com/</a></p>\n}
+    end
+
+    it 'should pass through URIs unchanged inside <nowiki></nowiki> spans' do
+      @parser.parse("<nowiki>http://example.com/</nowiki>").should == "<p>http://example.com/</p>\n"
+    end
+
+    it 'should autolink URIs inside <pre></pre> spans' do
+      input     = ' http://example.com/'
+      expected  = %Q{<pre><a href="http://example.com/" class="external">http://example.com/</a></pre>\n}
+      @parser.parse(input).should == expected
+      @parser.external_link_class = nil
+      expected  = %Q{<pre><a href="http://example.com/">http://example.com/</a></pre>\n}
+      @parser.parse(input).should == expected
+    end
+
+    it 'should convert emails into hyperlinks' do
+      uri = 'user@example.com'
+      @parser.parse(uri).should == %Q{<p><a href="mailto:user@example.com" class="mailto">user@example.com</a></p>\n}
+    end
+
+    it 'should apply the mailto CSS class if set' do
+      uri = 'user@example.com'
+      @parser.mailto_class = 'foo'
+      @parser.parse(uri).should == %Q{<p><a href="mailto:user@example.com" class="foo">user@example.com</a></p>\n}
+    end
+
+    it 'should apply no CSS if the mailto class is set to nil' do
+      uri = 'user@example.com'
+      @parser.mailto_class = nil
+      @parser.parse(uri).should == %Q{<p><a href="mailto:user@example.com">user@example.com</a></p>\n}
+    end
+
+    it 'should pass through emails unchanged inside <nowiki></nowiki> spans' do
+      @parser.parse("<nowiki>user@example.com</nowiki>").should == "<p>user@example.com</p>\n"  # was a crasher
+    end
+
+    it 'should pass through emails unchanged inside <pre></pre> blocks' do
+      @parser.parse(" user@example.com").should == "<pre>user@example.com</pre>\n"  # was a crasher
+    end
+  end
+
+  describe 'off' do
+    before do
+      @parser.autolink = false
+    end
+
+    it 'should accept "autolink = false"' do
+      @parser.autolink.should == false
+    end
+
+    it 'should not convert URIs into hyperlinks' do
+      @parser.parse('http://example.com/').should == "<p>http://example.com/</p>\n"
+    end
+
+    it 'should not convert emails into hyperlinks' do
+      @parser.parse('user@example.com').should == "<p>user@example.com</p>\n"
+    end
+  end
+end

data/spec/blockquote_spec.rb

@@ -0,0 +1,570 @@
+#!/usr/bin/env ruby
+# Copyright 2007-2008 Wincent Colaiuta
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require File.join(File.dirname(__FILE__), 'spec_helper.rb')
+require 'wikitext'
+
+describe Wikitext::Parser, 'standard blockquotes (">" in first column)' do
+  before do
+    @parser = Wikitext::Parser.new
+  end
+
+  it 'should treat ">" in first column as a blockquote marker' do
+    expected = dedent <<-END
+      <blockquote>
+        <p>foo</p>
+      </blockquote>
+    END
+    @parser.parse('>foo').should == expected
+  end
+
+  it 'should accept (and ignore) one optional space after the ">"' do
+    expected = dedent <<-END
+      <blockquote>
+        <p>foo</p>
+      </blockquote>
+    END
+    @parser.parse('> foo').should == expected
+  end
+
+  it 'should recognize consecutive ">" as continuance of blockquote section' do
+    expected = dedent <<-END
+      <blockquote>
+        <p>foo bar</p>
+      </blockquote>
+    END
+    @parser.parse("> foo\n> bar").should == expected
+  end
+
+  it 'should not give ">" special treatment when not on the far left' do
+    @parser.parse('foo > bar').should == "<p>foo &gt; bar</p>\n"
+  end
+
+  it 'should allow nesting of blockquotes' do
+    expected = dedent <<-END
+      <blockquote>
+        <blockquote>
+          <p>foo</p>
+        </blockquote>
+      </blockquote>
+    END
+    @parser.parse('> > foo').should == expected
+  end
+
+  it 'should allow opening of a nested blockquote after other content' do
+    expected = dedent <<-END
+      <blockquote>
+        <p>foo</p>
+        <blockquote>
+          <p>bar</p>
+        </blockquote>
+      </blockquote>
+    END
+    @parser.parse("> foo\n> > bar").should == expected
+  end
+
+  it 'should allow opening of a nested blockquote before other content' do
+    expected = dedent <<-END
+      <blockquote>
+        <blockquote>
+          <p>foo</p>
+        </blockquote>
+        <p>bar</p>
+      </blockquote>
+    END
+    @parser.parse("> > foo\n> bar").should == expected
+  end
+
+  it 'should accept an empty blockquote' do
+    expected = dedent <<-END
+      <blockquote>
+      </blockquote>
+    END
+    @parser.parse('>').should == expected
+  end
+
+  it 'should jump out of blockquote mode on seeing a normal line of text' do
+    expected =  dedent <<-END
+      <blockquote>
+        <p>foo</p>
+      </blockquote>
+      <p>bar</p>
+    END
+    @parser.parse("> foo\nbar").should == expected
+  end
+
+  it 'should allow nesting of h1 blocks' do
+    expected = dedent <<-END
+      <blockquote>
+        <h1>foo</h1>
+      </blockquote>
+    END
+    @parser.parse('> = foo =').should == expected
+  end
+
+  it 'should allow nesting of h2 blocks' do
+    expected =  dedent <<-END
+      <blockquote>
+        <h2>foo</h2>
+      </blockquote>
+    END
+    @parser.parse('> == foo ==').should == expected
+  end
+
+  it 'should allow nesting of h3 blocks' do
+    expected = dedent <<-END
+      <blockquote>
+        <h3>foo</h3>
+      </blockquote>
+    END
+    @parser.parse('> === foo ===').should == expected
+  end
+
+  it 'should allow nesting of h4 blocks' do
+    expected = dedent <<-END
+      <blockquote>
+        <h4>foo</h4>
+      </blockquote>
+    END
+    @parser.parse('> ==== foo ====').should == expected
+  end
+
+  it 'should allow nesting of h5 blocks' do
+    expected = dedent <<-END
+      <blockquote>
+        <h5>foo</h5>
+      </blockquote>
+    END
+    @parser.parse('> ===== foo =====').should == expected
+  end
+
+  it 'should allow nesting of h6 blocks' do
+     expected = dedent <<-END
+      <blockquote>
+        <h6>foo</h6>
+      </blockquote>
+    END
+    @parser.parse('> ====== foo ======').should == expected
+  end
+
+  it 'should allow alternating nested paragraphs and pre blocks' do
+    # was a bug
+    input = dedent <<-END
+      > para 1
+      >
+      >  pre 1
+      >  pre 2
+      >
+      > para 2
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>para 1</p>
+        <pre>pre 1
+      pre 2</pre>
+        <p>para 2</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should allow nesting of styled spans inside blockquotes' do
+    input = dedent <<-END
+      > link to [[something]], and ''other''
+      > `styled` '''spans'''.
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>link to <a href="/wiki/something">something</a>, and <em>other</em> <tt>styled</tt> <strong>spans</strong>.</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should allow complex nestings inside blockquotes' do
+    # was a bug: further reduced in the following example, "should handle TT spans inside blockquotes"
+    input = dedent <<-END
+       code block
+      
+      a normal para
+      
+      > will nest
+      >
+      >  line 1
+      >  line 2
+      >
+      > other
+      >
+      > * here
+      > * is
+      > * a
+      > * list
+      >
+      > outer para with ''styled''
+      > `stuff` in it
+      >
+      > > inner
+      > > blockquote
+      > > # inner list
+      > > ## nested list
+      
+      follow-up para
+    END
+    expected = dedent <<-END
+      <pre>code block</pre>
+      <p>a normal para</p>
+      <blockquote>
+        <p>will nest</p>
+        <pre>line 1
+      line 2</pre>
+        <p>other</p>
+        <ul>
+          <li>here</li>
+          <li>is</li>
+          <li>a</li>
+          <li>list</li>
+        </ul>
+        <p>outer para with <em>styled</em> <tt>stuff</tt> in it</p>
+        <blockquote>
+          <p>inner blockquote</p>
+          <ol>
+            <li>inner list
+              <ol>
+                <li>nested list</li>
+              </ol>
+            </li>
+          </ol>
+        </blockquote>
+      </blockquote>
+      <p>follow-up para</p>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should handle TT spans inside blockquotes' do
+    # was a bug: this is a minimally reduced test case extracted from the integration tests
+    input = dedent <<-END
+      > some
+      > `styled`
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>some <tt>styled</tt></p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+
+  it 'should handled nested lists which immediately follow paragraphs' do
+    # was a bug: this is a minimally reduced test case extracted from the integration tests
+    input = dedent <<-END
+      > Finally
+      > # Which
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>Finally</p>
+        <ol>
+          <li>Which</li>
+        </ol>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  # TODO: tests for nesting other types of blocks
+end
+
+describe Wikitext::Parser, 'literal BLOCKQUOTE_START/BLOCKQUOTE_END tags' do
+  before do
+    @parser = Wikitext::Parser.new
+  end
+
+  it 'should accept literal BLOCKQUOTE_START/BLOCKQUOTE_END tags as an alternative to the standard syntax' do
+    input = '<blockquote>hello</blockquote>'
+    expected = dedent <<-END
+      <blockquote>
+        <p>hello</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+
+    # alternative 1
+    input = dedent <<-END
+      <blockquote>
+      hello
+      </blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>hello</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+
+    # alternative 2
+    input = dedent <<-END
+      <blockquote>hello
+      </blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>hello</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+
+    # alternative 3
+    input = dedent <<-END
+      <blockquote>
+      hello</blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>hello</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+
+    # note what happens if we indent (whitespace gets carried through; it is not identified as a PRE block
+    # in reality you'd never indent when editing wikitext anyway; the idea is to free yourself from details like that
+    input = dedent <<-END
+      <blockquote>
+        hello
+      </blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>  hello</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should merge consecutive lines into a single paragraph' do
+    input = dedent <<-END
+      <blockquote>foo
+      bar
+      baz</blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>foo bar baz</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should process paragraph breaks' do
+    input = dedent <<-END
+      <blockquote>foo
+      
+      baz</blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>foo</p>
+        <p>baz</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should pass through PRE tokens unaltered' do
+    input = dedent <<-END
+      <blockquote>foo
+       bar</blockquote>
+    END
+
+    # note the extra space: one for the CRLF and another for the PRE token
+    expected = dedent <<-END
+      <blockquote>
+        <p>foo  bar</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should terminate open span-level elements on hitting the newline' do
+    # for now just test with EM; potentially add more examples later
+    input = dedent <<-END
+      <blockquote>foo ''bar
+      baz</blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>foo <em>bar</em> baz</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should pass through BLOCKQUOTE tokens escaped' do
+    input = dedent <<-END
+      <blockquote>foo
+      > bar
+      baz</blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>foo &gt; bar baz</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should be able to nest single-item unordered lists' do
+    input = '<blockquote>* foo</blockquote>'
+    expected = dedent <<-END
+      <blockquote>
+        <ul>
+          <li>foo</li>
+        </ul>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should be able to nest multi-item unordered lists' do
+    input = dedent <<-END
+      <blockquote>
+      * foo
+      * bar
+      * baz
+      </blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <ul>
+          <li>foo</li>
+          <li>bar</li>
+          <li>baz</li>
+        </ul>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should be able to nest nested unordered lists' do
+    input = dedent <<-END
+      <blockquote>
+      * foo
+      ** bar
+      * baz
+      </blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <ul>
+          <li>foo
+            <ul>
+              <li>bar</li>
+            </ul>
+          </li>
+          <li>baz</li>
+        </ul>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+
+    # note that the exact placement of the closing tag doesn't matter
+    input = dedent <<-END
+      <blockquote>
+      * foo
+      ** bar
+      * baz</blockquote>
+    END
+    @parser.parse(input).should == expected
+
+    # likewise for the opening tag
+    input = dedent <<-END
+      <blockquote>* foo
+      ** bar
+      * baz
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should be able to nest blockquotes' do
+    input = dedent <<-END
+      <blockquote>
+      foo
+      <blockquote>
+      bar
+      </blockquote>
+      baz
+      </blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>foo</p>
+        <blockquote>
+          <p>bar</p>
+        </blockquote>
+        <p>baz</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should be able to nest pre blocks' do
+    input = dedent <<-END
+      <blockquote>
+      outer 1
+      <pre>inner 1
+      inner 2</pre>
+      outer 2
+      </blockquote>
+    END
+    expected = dedent <<-END
+      <blockquote>
+        <p>outer 1</p>
+        <pre>inner 1
+      inner 2</pre>
+        <p>outer 2</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+
+  it 'should support nesting of H1 spans' do
+    input = dedent <<-END
+      <blockquote>
+      = foo =
+      bar
+      </blockquote>
+    END
+
+    expected = dedent <<-END
+      <blockquote>
+        <h1>foo</h1>
+        <p>bar</p>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+
+    # but note that this won't work
+    # the second "=" is not recognized as an H1_END because the scanner has no lookahead at the token level
+    input = '<blockquote>= foo =</blockquote>'
+    expected = dedent <<-END
+      <blockquote>
+        <h1>foo =</h1>
+      </blockquote>
+    END
+    @parser.parse(input).should == expected
+  end
+end