marker 0.1.0 → 0.2.0

data/lib/marker/language.treetop

@@ -183,7 +183,9 @@ module Marker
       #   graph
       # MediaWiki renders (newlines removed): <hr /><p>a para-</p><p>graph</p>
       # this renders (newlines removed): <hr /><p>a para-graph</p>
-      "----" "-"* ws text? <HorizRule>
+      "----" "-"* ws t:unsafe_text <HorizRule>
+      /
+      "----" "-"* <HorizRule>
     end
 
     ##### combination rules
@@ -195,6 +197,20 @@ module Marker
       !line h:phrase "" <Paragraph>
     end
 
+    # a block of text that does not require a !line check
+    #
+    # for situations where text may follow structures that are normally on a
+    # single line---e.g., horizontal rules--- the text should be handled like a
+    # paragraph on a new line, but using the text rule will cause errors when
+    # the text starts like a special line: cases like this: "---- ----".
+    # Because it follows a structure without a new line, it is safe to assume
+    # the text is a phrase.
+    rule unsafe_text
+      h:phrase ws rnl r:text <Paragraph>
+      /
+      h:phrase "" <Paragraph>
+    end
+
     # a phrase of words without newlines
     rule phrase
       h:word ws r:phrase <Phrase>
@@ -312,6 +328,7 @@ module Marker
     # [[ link target | link label ]]
     # * can contain white space
     # * cannot contain new lines
+    #-- TODO: handle [[ url | label ]] and variants ++
     rule internal_link
       internal_link_start ws t:plain_text ws a:arg_delimiter ws l:internal_link_enclosed_text ws internal_link_end <InternalLink>
       /
@@ -333,8 +350,6 @@ module Marker
       /
       template
       /
-      url
-      /
       plain_word
       /
       bold_toggle
@@ -366,15 +381,24 @@ module Marker
     # renders differently from MediaWiki:
     #   [ external [[internal]] ]
     # MediaWiki: <a href="external"></a><a href="internal">internal</a>
-    # This lib:  <a href="external">[[internal]]</a>
+    # Marker:  <a href="external">[[internal]]</a>
+    #
+    # also use this rule to match bad internal link syntax [[ url | label ]]
+    # because the rendering rules are the same
     rule external_link
-      external_link_start ws t:url rws l:external_link_enclosed_text ws external_link_end <ExternalLink>
-      /
-      external_link_start ws t:url ws external_link_end <ExternalLink>
-      /
+#      external_link_start ws t:url rws l:external_link_enclosed_text ws external_link_end <ExternalLink>
+#      /
+#      external_link_start ws t:url ws external_link_end <ExternalLink>
+#      /
       external_link_start ws t:implicit_url rws l:external_link_enclosed_text ws external_link_end <ExternalLink>
       /
       external_link_start ws t:implicit_url ws external_link_end <ExternalLink>
+      /
+      internal_link_start ws t:implicit_url ws arg_delimiter ws l:internal_link_enclosed_text ws internal_link_end <ExternalLink>
+      /
+      internal_link_start ws t:implicit_url rws l:internal_link_enclosed_text ws internal_link_end <ExternalLink>
+      /
+      internal_link_start ws t:implicit_url ws internal_link_end <ExternalLink>
     end
 
     rule external_link_enclosed_text
@@ -425,12 +449,6 @@ module Marker
       h:arg aws arg_delimiter aws r:arg_list <Arguments>
       /
       h:arg "" <Arguments>
-      /
-      aws "" <Arguments> {
-        def h
-          nil
-        end
-      }
     end
 
     # a plain (positional) argument or a named (name=text) argument
@@ -438,6 +456,8 @@ module Marker
       name:plain_text aws heading_toggle aws val:arg_list_enclosed_text <Argument>
       /
       val:arg_list_enclosed_text "" <Argument>
+      /
+      "" "" <Argument>
     end
 
     rule arg_list_enclosed_text
@@ -650,12 +670,16 @@ module Marker
       [A-Fa-f0-9]
     end
 
+    #-- TODO: Handle windows newlines (\r\n) ++
+
     # required new lines
+    # just one!
     rule rnl
-      "\n"+
+      "\n"
     end
 
     # new lines
+    #-- TODO: should this be broken into exactly one and any number rules? ++
     rule nl
       "\n"*
     end

data/lib/marker/links.rb

@@ -80,7 +80,7 @@ module Marker #:nodoc:
           n = f.add( target )
           "<a href='#{target}'>[#{n}]</a>"
         else
-          "<a href='#{target}'></a>"
+          "<a href='#{target}'>#{target}</a>"
         end
       end
     end
@@ -136,14 +136,28 @@ module Marker #:nodoc:
   end
 
   class URL < ParseNode
-    # TODO: these should output links just like "[url]".  The link tags
-    # shouldn't call these methods, these are called when the URL is not in a
-    # link.  The link templates should grab the URL themselves.
     def to_html( options = {} )
-      text_value
+      f = options[:footnotes]
+      if f
+        n = f.add( bare_url )
+        "<a href='#{bare_url}'>[#{n}]</a>"
+      else
+        "<a href='#{bare_url}'>#{bare_url}</a>"
+      end
     end
 
     def to_s( options = {} )
+      f = options[:footnotes]
+      if f
+        n = f.add( bare_url )
+        "[#{n}]"
+      else
+        bare_url
+      end
+    end
+
+    # returns just the URL that was matched
+    def bare_url
       text_value
     end
   end

data/lib/marker/templates.rb

@@ -48,19 +48,21 @@ module Marker #:nodoc:
       when :html
         to_a.each do |a|
           next unless a
+          value = ( a.val ? a.val.to_html(options) : "" )
           if a.name
-            named_params[a.name.to_html( options )] = a.val.to_html( options )
+            named_params[a.name.to_s(options)] = value
           else
-            pos_params << a.val.to_html( options )
+            pos_params << value
           end
         end
       else
         to_a.each do |a|
           next unless a
+          value = ( a.val ? a.val.to_s(options) : "" )
           if a.name
-            named_params[a.name.to_s( options )] = a.val.to_s( options )
+            named_params[a.name.to_s(options)] = value
           else
-            pos_params << a.val.to_s( options )
+            pos_params << value
           end
         end
       end
@@ -94,6 +96,10 @@ module Marker #:nodoc:
     def name #:nodoc:
       nil
     end
+
+    def val
+      nil
+    end
   end
 
   # A set of basic templates for rendering

data/lib/marker/text.rb

@@ -74,6 +74,23 @@ module Marker #:nodoc:
     end
   end
 
+  class HorizRule < ParseNode
+    def to_html( options = {} )
+      "<hr />" +
+      ( t ? "\n#{t.to_html(options)}" : "" )
+    end
+
+    def to_s( options = {} )
+      width = options[:width] || 80
+      "-" * width +
+      ( t ? "\n#{t.to_s(options)}" : "" )
+    end
+
+    def t
+      nil
+    end
+  end
+
   class Bold < ParseNode
     def to_html( options = {} )
       "<b>#{bold_enclosed_text.to_html(options)}</b>"

data/lib/marker/verbatim.rb

@@ -34,15 +34,4 @@ module Marker #:nodoc:
       v.text_value
     end
   end
-
-  class HorizRule < ParseNode
-    def to_html( options = {} )
-      "<hr />"
-    end
-
-    def to_s( options = {} )
-      width = options[:width] || 80
-      "-" * width
-    end
-  end
 end

data/test/formatting_test.rb

@@ -0,0 +1,98 @@
+require File.dirname(__FILE__) + '/test_helper'
+require 'marker'
+
+class FormattingTest < Test::Unit::TestCase
+
+  def test_bold_formatting
+    text = "'''bold'''"
+    markup = Marker.parse text
+    
+    assert_match("<p><b>bold</b></p>", markup.to_html)
+  end
+
+  def test_italic_formatting
+    text = "''italic''"
+    markup = Marker.parse text
+    
+    assert_match("<p><i>italic</i></p>", markup.to_html)
+  end
+
+  def test_bold_italic_formatting
+    # starting and finishing bold-italic
+    text = "'''''bold & italic'''''"
+    markup = Marker.parse text
+    assert_match("<p><b><i>bold & italic</i></b></p>", markup.to_html)
+
+    # starting italic, finishing bold-italic
+    text = "''italic'''bold & italic'''''"
+    markup = Marker.parse text
+    assert_match("<p><i>italic<b>bold & italic</b></i></p>", markup.to_html)
+
+    # starting bold, finishing bold-italic
+    text = "'''bold''bold & italic'''''"
+    markup = Marker.parse text
+    assert_match("<p><b>bold<i>bold & italic</i></b></p>", markup.to_html)
+
+    # starting bold-italic, finishing bold
+    text = "'''''bold & italic''bold'''"
+    markup = Marker.parse text
+    assert_match("<p><b><i>bold & italic</i>bold</b></p>", markup.to_html)
+
+    # starting bold-italic, finishing italic
+    #
+    # known issue: this doesn't work as "expected" due to treetop's
+    # short-circuiting (PEG); note that the opposite (bold outside italic)
+    # works fine
+    #
+    # TODO: maybe catch this case with a special grammar rule?
+    #
+    # doesn't work:
+    text = "'''''word'''word''"
+    markup = Marker.parse text
+    assert_match("<p>'''<i>word</i>'word''</p>", markup.to_html)
+    # works fine if given a space to be more clear:
+    text = "'' '''bold & italic'''italic''"
+    markup = Marker.parse text
+    assert_match("<p><i><b>bold & italic</b>italic</i></p>", markup.to_html)
+
+    # improper nesting
+    #
+    # marker will not catch improper nesting:
+    # * MediaWiki: <p><i>one<b>two</b></i><b>three</b></p>
+    # * Marker: <p><i>one</i>'two<i>three</i>'</p>
+    text = "''one'''two''three'''"
+    markup = Marker.parse text
+    assert_match("<p><i>one</i>'two<i>three</i>'</p>", markup.to_html)
+  end
+
+  def test_paragraphs
+    text = "paragraph 1\n\nparagraph 2\n\nparagraph 3"
+    markup = Marker.parse text
+
+    assert_match("<p>paragraph 1</p>\n\n<p>paragraph 2</p>\n\n<p>paragraph 3</p>", markup.to_html)
+  end
+
+  def test_newlines_in_paragraphs
+    # text lines separated by only one newline
+    # this matches how MediaWiki renders
+    text = "word1\nword2\nword3"
+    markup = Marker.parse text
+
+    assert_match("<p>word1 word2 word3</p>", markup.to_html)
+  end
+
+  def test_horizontal_rule
+    text = "----"
+    markup = Marker.parse text
+    
+    assert_match("<hr />\n", markup.to_html)
+  end
+
+  def test_invalid_horizontal_rule
+    text = "---- ----"
+    markup = Marker.parse text
+    
+    assert_match("<hr />\n<p>----</p>", markup.to_html)
+  end
+
+end

data/test/headings_test.rb

@@ -0,0 +1,77 @@
+require File.dirname(__FILE__) + '/test_helper'
+require 'marker'
+
+class HeadingTest < Test::Unit::TestCase
+
+  def test_level_one
+    text = "= Level 1 ="
+    markup = Marker.parse text
+
+    assert_match("<h1>Level 1</h1>", markup.to_html)
+  end
+
+  def test_level_two
+    text = "== Level 2 =="
+    markup = Marker.parse text
+
+    assert_match("<h2>Level 2</h2>", markup.to_html)
+  end
+
+  def test_level_three
+    text = "=== Level 3 ==="
+    markup = Marker.parse text
+
+    assert_match("<h3>Level 3</h3>", markup.to_html)
+  end
+
+  def test_level_four
+    text = "==== Level 4 ===="
+    markup = Marker.parse text
+
+    assert_match("<h4>Level 4</h4>", markup.to_html)
+  end
+
+  def test_level_five
+    text = "===== Level 5 ====="
+    markup = Marker.parse text
+
+    assert_match("<h5>Level 5</h5>", markup.to_html)
+  end
+
+  def test_level_six
+    text = "====== Level 6 ======"
+    markup = Marker.parse text
+
+    assert_match("<h6>Level 6</h6>", markup.to_html)
+  end
+
+  def test_unbalanced_delimiters
+    # this behavior should match MediaWiki
+    text = "=== Level 2 =="
+    markup = Marker.parse text
+    assert_match("<h2>= Level 2</h2>", markup.to_html)
+
+    text = "== Level 2 ==="
+    markup = Marker.parse text
+    assert_match("<h2>Level 2 =</h2>", markup.to_html)
+  end
+
+  def test_containing_equals
+    text = "== Level = 2 =="
+    markup = Marker.parse text
+
+    assert_match("<h2>Level = 2</h2>", markup.to_html)
+  end
+
+  def test_trailing_text
+    # headings must be alone on a line or else are treated as text; this
+    # behavior matches MediaWiki
+    #
+    # should this be changed to work like horizontal rules?
+    text = "== Level 2 == trailing text"
+    markup = Marker.parse text
+
+    assert_match("<p>== Level 2 == trailing text</p>", markup.to_html)
+  end
+
+end

data/test/links_test.rb

@@ -0,0 +1,109 @@
+require File.dirname(__FILE__) + '/test_helper'
+require 'marker'
+
+class LinkTest < Test::Unit::TestCase
+
+  def test_internal_link_with_alias
+    text = "[[example_page|Example page]]"
+    markup = Marker.parse text
+    
+    assert_match("<a href='/example_page'>Example page</a>", markup.to_html)
+  end
+
+  def test_bare_internal_link
+    text = "[[Example page]]"
+    markup = Marker.parse text
+    
+    assert_match("<a href='/Example page'>Example page</a>", markup.to_html)
+  end
+
+  def test_internal_link_with_sanitization
+    # parens aren't normally sanitized
+    text = "[[Example page (disambiguation)]]"
+    markup = Marker.parse text
+    assert_match("<a href='/Example page (disambiguation)'>Example page (disambiguation)</a>", markup.to_html)
+
+    # with a blank alias, parens are sanitized from the alias
+    text = "[[Example page (disambiguation)|]]"
+    markup = Marker.parse text
+    assert_match("<a href='/Example page (disambiguation)'>Example page</a>", markup.to_html)
+
+    # no effect without parens
+    text = "[[Example page|]]"
+    markup = Marker.parse text
+    assert_match("<a href='/Example page'>Example page</a>", markup.to_html)
+  end
+
+  def test_internal_with_url
+    text = "[[http://www.example.com]]"
+    markup = Marker.parse text
+    assert_match("<p><a href='http://www.example.com'>[1]</a></p>\n<ol><li><a href='http://www.example.com'>http://www.example.com</a></li></ol>", markup.to_html)
+
+    text = "[[http://www.example.com|Example page]]"
+    markup = Marker.parse text
+    assert_match("<a href='http://www.example.com'>Example page</a>", markup.to_html)
+
+    text = "[[http://www.example.com Example page]]"
+    markup = Marker.parse text
+    assert_match("<a href='http://www.example.com'>Example page</a>", markup.to_html)
+  end
+
+  def test_external_link_with_alias
+    text = "[http://www.example.com Example link]"
+    markup = Marker.parse text
+    
+    assert_match("<a href='http://www.example.com'>Example link</a>", markup.to_html)
+  end
+
+  def test_bare_external_link
+    text = "[http://www.example.com]"
+    markup = Marker.parse text
+
+    assert_match("<p><a href='http://www.example.com'>[1]</a></p>\n<ol><li><a href='http://www.example.com'>http://www.example.com</a></li></ol>", markup.to_html)
+  end
+
+  def test_nested_links
+    # only the outside link works
+    text = "[http://www.example.com [[ Nested Internal ]]]"
+    markup = Marker.parse text
+    assert_match("<p><a href='http://www.example.com'>[[ Nested Internal ]]</a></p>", markup.to_html)
+
+    text = "[[Internal | [http://www.example.com Nested] ]]"
+    markup = Marker.parse text
+    assert_match("<p><a href='/Internal'>[http://www.example.com Nested]</a></p>", markup.to_html)
+
+    text = "[[Internal | [http://www.example.com Nested]]]"
+    markup = Marker.parse text
+    assert_match("<p><a href='/Internal'>[http://www.example.com Nested</a>]</p>", markup.to_html)
+  end
+
+  def test_implicit_urls
+    # implicit urls are url-safe strings used in external links.  e.g., apt: or itunes: urls
+    # alone, they render as text:
+    text = "apt:ruby1.8"
+    markup = Marker.parse text
+    assert_match("<p>apt:ruby1.8</p>", markup.to_html)
+
+    # placed in a url context, they render as urls:
+    text = "[apt:ruby1.8 Install Ruby!]"
+    markup = Marker.parse text
+    assert_match("<a href='apt:ruby1.8'>Install Ruby!</a>", markup.to_html)
+  end
+
+  def test_bare_url
+    text = "http://www.example.com"
+    markup = Marker.parse text
+    
+    assert_match("<p><a href='http://www.example.com'>[1]</a></p>\n<ol><li><a href='http://www.example.com'>http://www.example.com</a></li></ol>", markup.to_html)
+  end
+
+  def test_multiple_footnotes
+    text = "http://www.example.com [http://www.example.com]"
+    markup = Marker.parse text
+
+    # TODO: should this collect identical links into one footnote?
+    assert_match("<p><a href='http://www.example.com'>[1]</a> <a href='http://www.example.com'>[2]</a></p>\n" +
+        "<ol><li><a href='http://www.example.com'>http://www.example.com</a></li><li><a href='http://www.example.com'>http://www.example.com</a></li></ol>", markup.to_html)
+  end
+
+end

data/test/lists_test.rb

@@ -0,0 +1,73 @@
+require File.dirname(__FILE__) + '/test_helper'
+require 'marker'
+
+class ListTest < Test::Unit::TestCase
+
+  def test_bulleted_list
+    text = "* List item 1\n* List item 2"
+    markup = Marker.parse text
+    
+    assert_match("<ul><li>List item 1</li><li>List item 2</li></ul>", markup.to_html)
+  end
+
+  def test_nested_bulleted_list
+    text = "* List item 1\n** List item 2\n* List item 3"
+    markup = Marker.parse text
+    
+    assert_match("<ul><li>List item 1</li><ul><li>List item 2</li></ul><li>List item 3</li></ul>", markup.to_html)
+  end
+
+  def test_numbered_list
+    text = "# List item 1\n# List item 2"
+    markup = Marker.parse text
+    
+    assert_match("<ol><li>List item 1</li><li>List item 2</li></ol>", markup.to_html)
+  end
+
+  def test_nested_numbered_list
+    text = "# List item 1\n## List item 2\n# List item 3"
+    markup = Marker.parse text
+    
+    assert_match("<ol><li>List item 1</li><ol><li>List item 2</li></ol><li>List item 3</li></ol>", markup.to_html)
+  end
+
+  def test_definition_list
+    text = "; term : definition"
+    markup = Marker.parse text
+
+    assert_match("<dl><dt>term</dt><dd>definition</dd></dl>", markup.to_html)
+  end
+
+  def test_nested_definition_list
+    # definition lists can't be nested, so it should return the markup as normal text
+    text = "; term 1 : ; term 2 : definition"
+    markup = Marker.parse text
+
+    assert_match("<dl><dt>term 1</dt><dd>; term 2 : definition</dd></dl>", markup.to_html)
+  end
+
+  def test_indented_list
+    # MediaWiki renders intented lists as definition lists without terms.
+    # Marker uses <div> tags instead
+  end
+
+  def test_nested_intented_list
+  end
+
+  def test_mixed_list
+    text = "# List item 1\n* List item 2\n# List item 3\n; List item 4 : definition\n: List item 5\n* List item 6"
+    markup = Marker.parse text
+    
+    assert_match("<ol><li>List item 1</li></ol><ul><li>List item 2</li></ul><ol><li>List item 3</li></ol><dl><dt>List item 4</dt>" +
+        "<dd>definition</dd></dl><div class='indent'><div>List item 5</div></div><ul><li>List item 6</li></ul>", markup.to_html)
+  end
+
+  def test_nested_mixed_list
+    text = "# List item 1\n#* List item 2\n# List item 3\n## List item 4\n#; List item 5 : definition\n#:List item 6"
+    markup = Marker.parse text
+    
+    assert_match("<ol><li>List item 1</li><ul><li>List item 2</li></ul><li>List item 3</li><ol><li>List item 4</li></ol>" +
+        "<dl><dt>List item 5</dt><dd>definition</dd></dl><div class='indent'><div>List item 6</div></div></ol>", markup.to_html)
+  end
+
+end

data/test/templates_test.rb

@@ -0,0 +1,52 @@
+require File.dirname(__FILE__) + '/test_helper'
+require 'marker'
+
+class TemplatesTest < Test::Unit::TestCase
+
+  def test_basic_template
+    text = "{{ template }}"
+    markup = Marker.parse text
+    
+    assert_match("<p>render:template( :html, [], {} )</p>", markup.to_html)
+  end
+
+  def test_positional_args
+    text = "{{ template | one | two | three }}"
+    markup = Marker.parse text
+    
+    assert_match("<p>render:template( :html, [\"one\", \"two\", \"three\"], {} )</p>", markup.to_html)
+  end
+
+  def test_named_args
+    text = "{{ template | one=1 | two = 2 }}"
+    markup = Marker.parse text
+    
+    # might want to fix this assertion, the hash could come out in any order
+    assert_match("<p>render:template( :html, [], {\"two\"=>\"2\", \"one\"=>\"1\"} )</p>", markup.to_html)
+  end
+
+  def test_mixed_args
+    text = "{{ template | one | two = 2 | three }}"
+    markup = Marker.parse text
+    
+    assert_match("<p>render:template( :html, [\"one\", \"three\"], {\"two\"=>\"2\"} )</p>", markup.to_html)
+  end
+
+  def test_empty_args
+    text = "{{ template | }}"
+    markup = Marker.parse text
+    assert_match("<p>render:template( :html, [\"\"], {} )</p>", markup.to_html)
+
+    text = "{{ template | | }}"
+    markup = Marker.parse text
+    assert_match("<p>render:template( :html, [\"\", \"\"], {} )</p>", markup.to_html)
+  end
+
+  def test_rendered_args
+    text = "{{ template | [http://www.example.com Example] }}"
+    markup = Marker.parse text
+
+    assert_match("<p>render:template( :html, [\"<a href='http://www.example.com'>Example</a>\"], {} )</p>", markup.to_html)
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,2 @@
+require 'test/unit'
+$LOAD_PATH << File.expand_path( File.dirname(__FILE__) + '/../lib' )

data/test/verbatim_test.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + '/test_helper'
+require 'marker'
+
+class VerbatimTest < Test::Unit::TestCase
+
+  def test_preformatted_text
+    text = " some source code"
+    markup = Marker.parse text
+    
+    assert_match("<pre>\nsome source code\n</pre>", markup.to_html)
+  end
+
+  def test_multiline_preformatted_text
+    text = " some source code\n some more source code"
+    markup = Marker.parse text
+    
+    assert_match("<pre>\nsome source code\nsome more source code\n</pre>", markup.to_html)
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: marker
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Ryan Blue
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-14 00:00:00 +01:00
+date: 2010-01-18 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -73,5 +73,11 @@ rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: A markup parser that outputs html and text.  Syntax is similar to MediaWiki.
-test_files: []
-
+test_files: 
+- test/templates_test.rb
+- test/verbatim_test.rb
+- test/headings_test.rb
+- test/formatting_test.rb
+- test/test_helper.rb
+- test/links_test.rb
+- test/lists_test.rb