mediacloth 0.0.3 → 0.5

data/README.md

@@ -0,0 +1,36 @@
+MediaCloth is the MediaWiki syntax parser and html generator written in ruby. It's small, fast and aims to recognize the complete MediaWiki language.
+
+## Installation
+To install the library run:
+
+    ruby setup.rb
+
+
+## Usage
+The quickest way to parse your input and produce html formatted text is:
+
+    require 'mediacloth'
+    puts MediaCloth::wiki_to_html("'''Hello'''''World''!")
+
+You can also provide a hash with custom options if you want to use another generator or link handler:
+
+    require 'mediacloth'
+    puts MediaCloth::wiki_to_html("'''Hello'''''World''!", :link_handler => MyLinkHandler.new)
+
+Both examples should produce
+
+    <b>Hello</b><i>World</i>!
+
+## API Docs
+To generate API documentation run:
+
+    rake rdoc
+
+## Development
+To run tests execute
+
+    rake test
+
+To regenerate test data (html and lex files from wiki input), run:
+
+    rake test:regenerate

data/lib/mediacloth/mediawikiast.rb

@@ -3,11 +3,15 @@ class AST
     attr_accessor :contents
     attr_accessor :parent
     attr_accessor :children
+    attr_accessor :index
+    attr_accessor :length
 
-    def initialize
+    def initialize(index = 0,length = 0)
         @children = []
         @parent = nil
         @contents = ""
+        @index = index
+        @length = length
     end
 end
 
@@ -20,6 +24,10 @@ end
 class ParagraphAST < AST
 end
 
+#The node to represent paragraph with text pasted into wiki
+class PasteAST < AST
+end
+
 #The node to represent a simple or formatted text
 #with more AST nodes inside.
 class FormattedAST < AST
@@ -36,6 +44,7 @@ end
 class LinkAST < AST
     #The link's URL
     attr_accessor :url
+    attr_accessor :link_type
 end
 
 #The node to represent a Mediawiki internal link
@@ -45,6 +54,13 @@ class InternalLinkAST < AST
     attr_accessor :locator
 end
 
+#The node to represent a Mediawiki category link
+class CategoryLinkAST < AST
+    #Holds the category locator, which is composed of a category name only
+    #(e.g. the name of the category)
+    attr_accessor :locator
+end
+
 #The node to represent a MediaWiki resource reference (embedded images, videos,
 #etc.)
 class ResourceLinkAST < AST
@@ -72,6 +88,7 @@ end
 class TableCellAST < AST
     #the type of cell, :head or :body
     attr_accessor :type
+    attr_accessor :attributes
 end
 
 #The node to represent a list
@@ -84,6 +101,14 @@ end
 class ListItemAST < AST
 end
 
+# The node to represent a leading term in a dictionary list
+class ListTermAST < AST
+end
+
+# The node to represent a definition in a dictionary list
+class ListDefinitionAST < AST
+end
+
 #The node to represent a section
 class SectionAST < AST
     #The level of the section (1,2,3...) that would correspond to
@@ -93,4 +118,36 @@ end
 
 #The node to represent a preformatted contents
 class PreformattedAST < AST
+    attr_accessor :indented
+end
+
+#The node to represent an XHTML element and its contents
+class ElementAST < AST
+    attr_accessor :name, :attributes
+end
+
+# The node to represent special Mediawiki keywords, such as __TOC__. The text
+# attribute contains the entire string inbetween '__' and '__'.
+class KeywordAST < AST
+    attr_accessor :text
+end
+
+# The node to represent templates and pre-defined (or user-defined) variables, such as
+# {{Date}}.
+class TemplateAST < AST
+    attr_accessor :template_name
+end
+
+# The node to represent template parameter
+class TemplateParameterAST < AST
+    attr_accessor :parameter_name   #not used atm
+    attr_accessor :parameter_value
+end
+
+#The node to represent categories to which this page belongs
+class CategoryAST < AST
+    #Holds the name of the category
+    attr_accessor :locator
+    #Holds the string the page is to be sorted as
+    attr_accessor :sort_as
 end

data/lib/mediacloth/mediawikihtmlgenerator.rb

@@ -11,55 +11,37 @@ require 'mediacloth/mediawikiparams'
 # walker.parse(ast)
 # puts walker.html
 class MediaWikiHTMLGenerator < MediaWikiWalker
+  
     attr_reader :html
 
-    def initialize
-        @html = ""
-    end
-
     def parse(ast)
+        @html = ""
+        @ast = ast
         @html = super(ast)
     end
     
-    #The default link handler. A custom link handler may extend this class.
-    class MediaWikiLinkHandler
-      
-      #Method invoked to resolve references to wiki pages when they occur in an
-      #internal link. In all the following internal links, the page name is
-      #<tt>My Page</tt>:
-      #* <tt>[[My Page]]</tt>
-      #* <tt>[[My Page|Click here to view my page]]</tt>
-      #* <tt>[[My Page|Click ''here'' to view my page]]</tt>
-      #The return value should be a URL that references the page resource.
-      def url_for(resource)
-        "javascript:void(0)"
-      end
-      
-      #Method invoked to resolve references to resources of unknown types. The
-      #type is indicated by the resource prefix. Examples of inline links to
-      #unknown references include:
-      #* <tt>[[Media:video.mpg]]</tt> (prefix <tt>Media</tt>, resource <tt>video.mpg</tt>)
-      #* <tt>[[Image:pretty.png|100px|A ''pretty'' picture]]</tt> (prefix <tt>Image</tt>,
-      #  resource <tt>pretty.png</tt>, and options <tt>100px</tt> and <tt>A
-      #  <i>pretty</i> picture</tt>.
-      #The return value should be a well-formed hyperlink, image, object or 
-      #applet tag.
-      def link_for(prefix, resource, options=[])
-        "<a href=\"javascript:void(0)\">#{prefix}:#{resource}(#{options.join(', ')})</a>"
-      end
-    end
-    
-    #Set this generator's URL handler.
-    def link_handler=(handler)
-      @link_handler = handler
+    # Utility method that returns the string with '<', '>', '&' and '"' escaped as 
+    # XHTML character entities
+    def MediaWikiHTMLGenerator.escape(str)
+        r = str.gsub(%r{[<>&"]}) do
+            |match|
+            case match
+            when '<' then '&lt;'
+            when '>' then '&gt;'
+            when '&' then '&amp;'
+            when '"' then '&quot;'
+            end
+        end
+        r
     end
     
-    #Returns's this generator URL handler. If no handler was set, returns the default
-    #handler.
-    def link_handler
-      @link_handler ||= MediaWikiLinkHandler.new
+    # Utility method that converts the string specified into a specially formatted text
+    # string which can be used as an XHTML link anchor name.
+    def MediaWikiHTMLGenerator.anchor_for(str)
+      str.strip.squeeze(' ').gsub(' ', '_').gsub('\'', '_')
     end
 
+    
 protected
 
     def parse_wiki_ast(ast)
@@ -67,20 +49,39 @@ protected
     end
 
     def parse_paragraph(ast)
-        "<p>" + super(ast) + "</p>"
+        if (children = ast.children)
+          if children.size == 1 and ((text = children.first.contents) == "\n\n" || text == "\r\n\r\n")
+            "<p><br />#{text}</p>"
+          else
+            "<p>#{super(ast)}</p>"
+          end
+        else
+            "<p><br /></p>"
+        end
+    end
+
+    def parse_paste(ast)
+        return '' unless ast.children
+        "<div class=\"paste\" style=\"white-space: pre-wrap;\">#{super(ast)}</div>"
     end
 
     def parse_text(ast)
-        tag = formatting_to_tag(ast)
-        if tag[0].empty?
-            ast.contents
+        if ast.formatting
+            case(ast.formatting)
+            when :None then MediaWikiHTMLGenerator.escape(ast.contents)
+            when :CharacterEntity then "&#{ast.contents};"
+            when :HLine then "<hr/>"
+            when :SignatureDate then @params.time.to_s
+            when :SignatureName then link_handler.link_for("User:#{@params.author}", @params.author)
+            when :SignatureFull then "#{link_handler.link_for("User:#{@params.author}", @params.author)} #{@params.time.to_s}"
+            end
         else
-            "<#{tag[0]}#{tag[1]}>#{ast.contents}</#{tag[0]}>"
+            escape(ast.contents)
         end
     end
 
     def parse_formatted(ast)
-        tag = formatting_to_tag(ast)
+        tag = ast.formatting == :Bold ? 'b' : 'i'
         "<#{tag}>" + super(ast) + "</#{tag}>"
     end
 
@@ -95,28 +96,74 @@ protected
         "<li>" + super(ast) + "</li>"
     end
 
+    def parse_list_term(ast)
+        "<dt>" + super(ast) + "</dt>"
+    end
+
+    def parse_list_definition(ast)
+        "<dd>" + super(ast) + "</dd>"
+    end
+
     def parse_preformatted(ast)
+        if ast.indented 
+            original_text = super(ast)
+            lines = original_text.split("\n").sort
+            shortest_space = lines.last.scan(/^\s+/)[0]
+            contents = ""
+            if shortest_space
+                original_text.each_line do |line|
+                    contents << line.sub(shortest_space, "")
+                end
+            else
+                contents = original_text
+            end
+            "<pre class=\"indent\">" + contents + "</pre>"
+        else
+            "<pre>" + super(ast) + "</pre>"
+        end
     end
 
     def parse_section(ast)
-        "<h#{ast.level}>" + super(ast) + "</h#{ast.level}>"
+        generator = TextGenerator.new
+        anchor = MediaWikiHTMLGenerator.anchor_for(generator.parse(ast).join(' '))
+        "<h#{ast.level}><a name='#{anchor}'></a>" + super(ast) + "</h#{ast.level}>\n"
     end
     
     def parse_internal_link(ast)
         text = parse_wiki_ast(ast)
-        text = ast.locator if text.length == 0
-        href = link_handler.url_for(ast.locator)
-        "<a href=\"#{href}\">#{text}</a>"
+        text = MediaWikiHTMLGenerator.escape(ast.locator) if text.length == 0
+        link_handler.link_for(ast.locator, text)
     end
-     
+
     def parse_resource_link(ast)
         options = ast.children.map do |node|
             parse_internal_link_item(node)
         end
-        link_handler.link_for(ast.prefix, ast.locator, options)
+        link_handler.link_for_resource(ast.prefix, ast.locator, options)
+    end
+
+    def parse_template(ast)
+        parameters = ast.children.map do |node|
+            if node.parameter_value
+                node.parameter_value
+            else
+                parse_template(node.children.first)
+            end
+        end
+        template_handler.included_template(ast.template_name, parameters)
+    end
+
+    def parse_category_link(ast)
+        text = parse_wiki_ast(ast)
+        text = MediaWikiHTMLGenerator.escape(ast.locator) if text.length == 0
+        link_handler.link_for_category(ast.locator, text)
+    end
+
+    def parse_category(ast)
+      text = parse_wiki_ast(ast)
+      link_handler.category_add(ast.locator, ast.sort_as)
     end
 
-    #Reimplement this
     def parse_internal_link_item(ast)
         text = super(ast)
         text.strip
@@ -125,13 +172,15 @@ protected
     def parse_link(ast)
         text = super(ast)
         href = ast.url
-        text = href if text.length == 0
-        "<a href=\"#{href}\">#{text}</a>"
+        text = MediaWikiHTMLGenerator.escape(href) if text.length == 0
+        link_handler.absolute_link_for(href, text, ast.link_type)
     end
 
     #Reimplement this
     def parse_table(ast)
         options = ast.options ? ' ' + ast.options.strip : ''
+        options << ' cellpadding="5"' unless options.include?('cellpadding')
+        options << ' border="1"' unless options.include?('border')
         "<table#{options}>" + super(ast) + "</table>\n"
     end
 
@@ -146,28 +195,33 @@ protected
         if ast.type == :head
             "<th>" + super(ast) + "</th>"
         else
-            "<td>" + super(ast) + "</td>"
+            if ast.attributes
+                "<td #{ast.attributes.first.contents}>" + super(ast) + "</td>"
+            else
+                "<td>" + super(ast) + "</td>"
+            end
         end
     end
 
-    #returns an array with a tag name and tag attributes
-    def formatting_to_tag(ast)
-        tag = ["", ""]
-        if ast.formatting == :Bold
-            tag = ["b", ""]
-        elsif ast.formatting == :Italic
-            tag = ["i", ""]
-        elsif ast.formatting == :HLine
-            ast.contents = ""
-            tag = ["hr", ""]
-        elsif ast.formatting == :SignatureDate
-            ast.contents = MediaWikiParams.instance.time.to_s
-        elsif ast.formatting == :SignatureName
-            ast.contents = MediaWikiParams.instance.author
-        elsif ast.formatting == :SignatureFull
-            ast.contents = MediaWikiParams.instance.author + " " + MediaWikiParams.instance.time.to_s
-        end
-        tag
+    def parse_element(ast)
+      attr = ''
+      if ast.attributes
+        attr = ' ' + ast.attributes.collect{ |name, value|
+          name + '="' + MediaWikiHTMLGenerator.escape(value) + '"' }.join(' ')
+      end
+      if ast.children.size == 0
+        "<#{ast.name}#{attr} />"
+      else
+        "<#{ast.name}#{attr}>" + super(ast) + "</#{ast.name}>"
+      end
+    end
+    
+    def parse_keyword(ast)
+      if ast.text == 'TOC'
+        generator = TocGenerator.new
+        generator.parse(@ast)
+        generator.html
+      end
     end
 
     #returns a tag name of the list in ast node
@@ -176,7 +230,109 @@ protected
             return "ul"
         elsif ast.list_type == :Numbered
             return "ol"
+        elsif ast.list_type == :Dictionary
+            return "dl"
+        end
+    end
+
+    # AST walker that generates a table of contents, containing links to all
+    # section headings in the page.
+    class TocGenerator < MediaWikiHTMLGenerator
+
+        class TocNode
+            attr_accessor :children
+            attr_accessor :parent
+            attr_accessor :section
+            def initialize
+                @children = []
+            end
+
+            def add_child(child)
+                @children << child
+                child.parent = self
+            end
+
+            def level
+                res = 0
+                node = self
+                while p = node.parent
+                    res += 1
+                    node = p
+                end
+                res
+            end
+
+            def number
+                res = ''
+                node = self
+                while p = node.parent
+                    res = "#{p.children.index(node)+1}." + res
+                    node = p
+                end
+                res
+            end
         end
+
+        def parse(ast)
+            @html = ''
+            @text_generator = TextGenerator.new
+
+            root = TocNode.new
+            root_stack = [root]
+
+            parse_branch = lambda do |ast| 
+                ast.children.each do |child|
+                    if child.class == SectionAST
+                        root_stack.pop while child.level <= ((sec = root_stack.last.section) ? sec.level : 0)
+
+                        node = TocNode.new
+                        node.section = child
+                        root_stack.last.add_child(node)
+
+                        root_stack.push node
+                    end
+                    parse_branch.call(child)
+                end
+            end
+            parse_branch.call(ast)
+
+            @html += parse_section(root)
+            @html = "<div class=\"wikitoc\">\n<div class=\"wikitoctitle\">Contents</div>#{@html}\n</div>\n" if @html != ''
+        end
+
+        protected
+
+        def parse_section(toc_node)
+            html = ''
+            if toc_node.section
+                anchor = MediaWikiHTMLGenerator.anchor_for(@text_generator.parse(toc_node.section).join(' '))
+                html += "\n<li><a href='##{anchor}'><span class=\"wikitocnumber\">#{toc_node.number}</span><span class=\"wikitoctext\">#{parse_wiki_ast(toc_node.section).strip}</span></a>"
+            end
+
+            unless toc_node.children.empty?
+                html += "\n<ul>"
+                toc_node.children.each do |child_node|
+                    html += parse_section(child_node)
+                end
+                html += "\n</ul>"
+            end
+
+            html += "</li>" if html[0,4] == "<li>"
+            html
+        end
+
+    end
+
+
+    # AST walker that outputs just the text portions of a page.
+    class TextGenerator < MediaWikiWalker
+
+        protected
+        
+        def parse_text(ast)
+            MediaWikiHTMLGenerator.escape(ast.contents)
+        end
+    
     end
 
 end