mediacloth 0.0.3 → 0.5

data/lib/mediacloth/mediawikiparser.y

@@ -8,13 +8,14 @@
 # parser.parse(input)
 class MediaWikiParser
 
-token BOLDSTART BOLDEND ITALICSTART ITALICEND LINKSTART LINKEND LINKSEP
-    INTLINKSTART INTLINKEND INTLINKSEP RESOURCE_SEP
-    SECTION_START SECTION_END TEXT PRE
-    HLINE SIGNATURE_NAME SIGNATURE_DATE SIGNATURE_FULL
-    UL_START UL_END LI_START LI_END OL_START OL_END
+token TEXT BOLD_START BOLD_END ITALIC_START ITALIC_END LINK_START LINK_END LINKSEP
+    INTLINK_START INTLINK_END INTLINKSEP RESOURCESEP CHAR_ENT 
+    PRE_START PRE_END PREINDENT_START PREINDENT_END 
+    SECTION_START SECTION_END HLINE SIGNATURE_NAME SIGNATURE_DATE SIGNATURE_FULL
+    PARA_START PARA_END UL_START UL_END OL_START OL_END LI_START LI_END
+    DL_START DL_END DT_START DT_END DD_START DD_END TAG_START TAG_END ATTR_NAME ATTR_VALUE
     TABLE_START TABLE_END ROW_START ROW_END HEAD_START HEAD_END CELL_START CELL_END
-    PARA_START PARA_END
+    KEYWORD TEMPLATE_START TEMPLATE_END CATEGORY PASTE_START PASTE_END
 
 
 rule
@@ -22,7 +23,7 @@ rule
 wiki:
     repeated_contents
         {
-            @nodes.push WikiAST.new
+            @nodes.push WikiAST.new(0, @wiki_ast_length)
             #@nodes.last.children.insert(0, val[0])
             #puts val[0]
             @nodes.last.children += val[0]
@@ -42,50 +43,87 @@ contents:
         {
             result = val[0]
         }
+    | dictionary_list
+        {
+            list = ListAST.new(@ast_index, @ast_length)
+            list.list_type = :Dictionary
+            list.children = val[0]
+            result = list
+        }
     | preformatted
         {
-            p = PreformattedAST.new
-            p.contents = val[0]
-            result = p
+            result = val[0]
         }
     | section
         {
             result = val[0]
         }
+    | tag
+        {
+            result = val[0]
+        } 
+    | template
+        {
+            result = val[0]
+        }
+    | KEYWORD
+        {
+            k = KeywordAST.new(@ast_index, @ast_length)
+            k.text = val[0]
+            result = k
+        }
     | PARA_START para_contents PARA_END
         {
-            if val[1]
-                p = ParagraphAST.new
-                p.children = val[1]
-                result = p
-            end
+            p = ParagraphAST.new(@ast_index, @ast_length)
+            p.children = val[1]
+            result = p
         }
-    | LINKSTART link_contents LINKEND
+    | LINK_START link_contents LINK_END
         {
-            l = LinkAST.new
+            l = LinkAST.new(@ast_index, @ast_length)
+            l.link_type = val[0]
             l.url = val[1][0]
             l.children += val[1][1..-1] if val[1].length > 1
             result = l
         }
-    | INTLINKSTART TEXT RESOURCE_SEP TEXT reslink_repeated_contents INTLINKEND
+    | PASTE_START para_contents PASTE_END
+        {
+            p = PasteAST.new(@ast_index, @ast_length)
+            p.children = val[1]
+            result = p
+        }
+    | INTLINK_START TEXT RESOURCESEP TEXT reslink_repeated_contents INTLINK_END
         {
-            l = ResourceLinkAST.new
+            l = ResourceLinkAST.new(@ast_index, @ast_length)
             l.prefix = val[1]
             l.locator = val[3]
             l.children = val[4] unless val[4].nil? or val[4].empty?
             result = l
         }
-    | INTLINKSTART TEXT intlink_repeated_contents INTLINKEND
+    | INTLINK_START TEXT intlink_repeated_contents INTLINK_END
         {
-            l = InternalLinkAST.new
+            l = InternalLinkAST.new(@ast_index, @ast_length)
             l.locator = val[1]
             l.children = val[2] unless val[2].nil? or val[2].empty?
             result = l
         }
+    | INTLINK_START CATEGORY TEXT cat_sort_contents INTLINK_END
+        {
+            l = CategoryAST.new(@ast_index, @ast_length)
+            l.locator = val[2]
+            l.sort_as = val[3]
+            result = l
+        }
+    | INTLINK_START RESOURCESEP CATEGORY TEXT intlink_repeated_contents INTLINK_END
+        {
+            l = CategoryLinkAST.new(@ast_index, @ast_length)
+            l.locator = val[3]
+            l.children = val[4] unless val[4].nil? or val[4].empty?
+            result = l
+        }
     | table
     ;
 
-#TODO: remove empty paragraphs in lexer
 para_contents: 
         {
             result = nil
@@ -96,6 +134,49 @@ para_contents:
         }
     ;
 
+tag:
+      TAG_START tag_attributes TAG_END 
+        {
+            if val[0] != val[2] 
+                raise Racc::ParseError.new("XHTML end tag #{val[2]} does not match start tag #{val[0]}")
+            end
+            elem = ElementAST.new(@ast_index, @ast_length)
+            elem.name = val[0]
+            elem.attributes = val[1]
+            result = elem
+        }
+    | TAG_START tag_attributes repeated_contents TAG_END 
+        {
+            if val[0] != val[3] 
+                raise Racc::ParseError.new("XHTML end tag #{val[3]} does not match start tag #{val[0]}")
+            end
+            elem = ElementAST.new(@ast_index, @ast_length)
+            elem.name = val[0]
+            elem.attributes = val[1]
+            elem.children += val[2]
+            result = elem
+        }
+    ;
+
+tag_attributes:
+        {
+            result = nil
+        }
+    | ATTR_NAME tag_attributes
+        {
+            attr_map = val[2] ? val[2] : {}
+            attr_map[val[0]] = true
+            result = attr_map 
+        }
+    | ATTR_NAME ATTR_VALUE tag_attributes
+        {
+            attr_map = val[2] ? val[2] : {}
+            attr_map[val[0]] = val[1]
+            result = attr_map 
+        }
+    ;
+      
+
 link_contents:
       TEXT
         {
@@ -132,6 +213,16 @@ intlink_repeated_contents:
         }
     ;
 
+cat_sort_contents:
+        {
+            result = nil
+        }
+    | INTLINKSEP TEXT
+        {
+            result = val[1]
+        }
+    ;
+
 reslink_repeated_contents:
         {
             result = nil
@@ -142,7 +233,7 @@ reslink_repeated_contents:
         }
     | INTLINKSEP repeated_contents reslink_repeated_contents
         {
-            i = InternalLinkItemAST.new
+            i = InternalLinkItemAST.new(@ast_index, @ast_length)
             i.children = val[1]
             result = [i]
             result += val[2] if val[2]
@@ -164,7 +255,7 @@ repeated_contents: contents
 
 text: element
         {
-            p = TextAST.new
+            p = TextAST.new(@ast_index, @ast_length)
             p.formatting = val[0][0]
             p.contents = val[0][1]
             result = p
@@ -178,13 +269,13 @@ text: element
 table:
       TABLE_START table_contents TABLE_END
         {
-            table = TableAST.new
+            table = TableAST.new(@ast_index, @ast_length)
             table.children = val[1] unless val[1].nil? or val[1].empty?
             result = table
         }
     | TABLE_START TEXT table_contents TABLE_END
         {
-            table = TableAST.new
+            table = TableAST.new(@ast_index, @ast_length)
             table.options = val[1]
             table.children = val[2] unless val[2].nil? or val[2].empty?
             result = table
@@ -196,14 +287,14 @@ table_contents:
         }
     | ROW_START row_contents ROW_END table_contents
         {
-            row = TableRowAST.new
+            row = TableRowAST.new(@ast_index, @ast_length)
             row.children = val[1] unless val[1].nil? or val[1].empty?
             result = [row]
             result += val[3] unless val[3].nil? or val[3].empty?
         }
     | ROW_START TEXT row_contents ROW_END table_contents
         {
-            row = TableRowAST.new
+            row = TableRowAST.new(@ast_index, @ast_length)
             row.children = val[2] unless val[2].nil? or val[2].empty?
             row.options = val[1]
             result = [row]
@@ -216,14 +307,14 @@ row_contents:
         }
     | HEAD_START HEAD_END row_contents
         {
-            cell = TableCellAST.new
+            cell = TableCellAST.new(@ast_index, @ast_length)
             cell.type = :head
             result = [cell]
             result += val[2] unless val[2].nil? or val[2].empty?
         }
     | HEAD_START repeated_contents HEAD_END row_contents
         {
-            cell = TableCellAST.new
+            cell = TableCellAST.new(@ast_index, @ast_length)
             cell.children = val[1] unless val[1].nil? or val[1].empty?
             cell.type = :head
             result = [cell]
@@ -231,26 +322,36 @@ row_contents:
         }
     | CELL_START CELL_END row_contents
         {
-            cell = TableCellAST.new
+            cell = TableCellAST.new(@ast_index, @ast_length)
             cell.type = :body
             result = [cell]
             result += val[2] unless val[2].nil? or val[2].empty?
         }
     | CELL_START repeated_contents CELL_END row_contents
         {
-            cell = TableCellAST.new
-            cell.children = val[1] unless val[1].nil? or val[1].empty?
-            cell.type = :body
-            result = [cell]
+            if val[2] == 'attributes'
+                result = []
+            else
+                cell = TableCellAST.new(@ast_index, @ast_length)
+                cell.children = val[1] unless val[1].nil? or val[1].empty?
+                cell.type = :body
+                result = [cell]
+            end
             result += val[3] unless val[3].nil? or val[3].empty?
+            if val[2] == 'attributes' and val[3] and val[3].first.class == TableCellAST
+                val[3].first.attributes = val[1]
+            end
+            result
         }
-    
+
 
 element:
       TEXT
         { return [:None, val[0]] }
     | HLINE
         { return [:HLine, val[0]] }
+    | CHAR_ENT
+        { return [:CharacterEntity, val[0]] }
     | SIGNATURE_DATE
         { return [:SignatureDate, val[0]] }
     | SIGNATURE_NAME
@@ -260,28 +361,28 @@ element:
     ;
 
 formatted_element: 
-      BOLDSTART BOLDEND
+      BOLD_START BOLD_END
         {
-            result = FormattedAST.new
+            result = FormattedAST.new(@ast_index, @ast_length)
             result.formatting = :Bold
             result
         } 
-    | ITALICSTART ITALICEND
+    | ITALIC_START ITALIC_END
         {
-            result = FormattedAST.new
+            result = FormattedAST.new(@ast_index, @ast_length)
             result.formatting = :Italic
             result
         }
-    | BOLDSTART repeated_contents BOLDEND
+    | BOLD_START repeated_contents BOLD_END
         {
-            p = FormattedAST.new
+            p = FormattedAST.new(@ast_index, @ast_length)
             p.formatting = :Bold
             p.children += val[1]
             result = p
         }
-    | ITALICSTART repeated_contents ITALICEND
+    | ITALIC_START repeated_contents ITALIC_END
         {
-            p = FormattedAST.new
+            p = FormattedAST.new(@ast_index, @ast_length)
             p.formatting = :Italic
             p.children += val[1]
             result = p
@@ -290,7 +391,7 @@ formatted_element:
 
 bulleted_list: UL_START list_item list_contents UL_END
         {
-            list = ListAST.new
+            list = ListAST.new(@ast_index, @ast_length)
             list.list_type = :Bulleted
             list.children << val[1]
             list.children += val[2]
@@ -300,7 +401,7 @@ bulleted_list: UL_START list_item list_contents UL_END
 
 numbered_list: OL_START list_item list_contents OL_END
         {
-            list = ListAST.new
+            list = ListAST.new(@ast_index, @ast_length)
             list.list_type = :Numbered
             list.children << val[1]
             list.children += val[2]
@@ -322,29 +423,116 @@ list_contents:
 list_item: 
       LI_START LI_END
         {
-            result = ListItemAST.new
+            result = ListItemAST.new(@ast_index, @ast_length)
         }
     | LI_START repeated_contents LI_END
         {
-            li = ListItemAST.new
+            li = ListItemAST.new(@ast_index, @ast_length)
             li.children += val[1]
             result = li
         }
     ;
 
-preformatted: PRE
-        { result = val[0] }
+dictionary_list:  
+      DL_START dictionary_term dictionary_contents DL_END
+        {
+            result = [val[1]]
+            result += val[2]
+        }
+    | DL_START dictionary_contents DL_END
+        {
+            result = val[1]
+        }
+    ;
+
+dictionary_term:
+      DT_START DT_END
+        {
+            result = ListTermAST.new(@ast_index, @ast_length)
+        }
+    | DT_START repeated_contents DT_END
+        {
+            term = ListTermAST.new(@ast_index, @ast_length)
+            term.children += val[1]
+            result = term
+        }
+
+dictionary_contents:
+      dictionary_definition dictionary_contents
+        {
+            result = [val[0]]
+            result += val[1] if val[1]
+        }
+    |
+        {
+            result = []
+        }
+
+dictionary_definition:
+      DD_START DD_END
+        {
+            result = ListDefinitionAST.new(@ast_index, @ast_length)
+        }
+    | DD_START repeated_contents DD_END
+        {
+            term = ListDefinitionAST.new(@ast_index, @ast_length)
+            term.children += val[1]
+            result = term
+        }
+
+preformatted: PRE_START repeated_contents PRE_END
+        {
+            p = PreformattedAST.new(@ast_index, @ast_length)
+            p.children += val[1]
+            result = p
+        }
+        | PREINDENT_START repeated_contents PREINDENT_END
+        {
+            p = PreformattedAST.new(@ast_index, @ast_length)
+            p.indented = true
+            p.children += val[1]
+            result = p
+        }
     ;
 
 section: SECTION_START repeated_contents SECTION_END
         { result = [val[1], val[0].length] 
-            s = SectionAST.new
+            s = SectionAST.new(@ast_index, @ast_length)
             s.children = val[1]
             s.level = val[0].length
             result = s
         }
     ;
 
+template: TEMPLATE_START TEXT template_parameters TEMPLATE_END
+        {
+            t = TemplateAST.new(@ast_index, @ast_length)
+            t.template_name = val[1]
+            t.children = val[2] unless val[2].nil? or val[2].empty?
+            result = t
+        }
+    ;
+
+template_parameters:
+        {
+            result = nil
+        }
+        | INTLINKSEP TEXT template_parameters
+        {
+            p = TemplateParameterAST.new(@ast_index, @ast_length)
+            p.parameter_value = val[1]
+            result = [p]
+            result += val[2] if val[2]
+        }
+        | INTLINKSEP template template_parameters
+        {
+            p = TemplateParameterAST.new(@ast_index, @ast_length)
+            p.children << val[1]
+            result = [p]
+            result += val[2] if val[2]
+        }
+    ;
+
 end
 
 ---- header ----
@@ -356,6 +544,8 @@ attr_accessor :lexer
 
 def initialize
     @nodes = []
+    @context = []
+    @wiki_ast_length = 0
     super
 end
 
@@ -369,5 +559,19 @@ end
 
 #Asks the lexer to return the next token.
 def next_token
-    return @lexer.lex
+    token = @lexer.lex
+    if token[0].to_s.upcase.include? "_START"
+        @context << token[2..3]
+    elsif token[0].to_s.upcase.include? "_END"
+        @ast_index = @context.last[0]
+        @ast_length = token[2] + token[3] - @context.last[0]
+        @context.pop
+    else
+        @ast_index = token[2]
+        @ast_length = token[3]
+    end
+    
+    @wiki_ast_length += token[3]
+    
+    return token[0..1]
 end

data/lib/mediacloth/mediawikisignedwikigenerator.rb

@@ -0,0 +1,42 @@
+require 'mediacloth/mediawikiwalker'
+require 'mediacloth/mediawikiparams'
+
+class MediaWikiSignedWikiGenerator < MediaWikiWalker
+    
+    attr_reader :signed_wiki
+    
+    def parse(ast, wiki)
+        @signed_wiki = wiki
+        #For supporting multiple signatures in wiki
+        @index_inc = 0
+        super(ast)
+    end
+
+protected
+
+    def parse_wiki_ast(ast)
+        super(ast).join
+    end
+
+    def parse_text(ast)
+        if ast.formatting
+            case(ast.formatting)
+            when :SignatureDate then
+                signature = @params.time
+                @signed_wiki[ast.index + @index_inc, ast.length] = signature
+                @index_inc += signature.length - ast.length
+            when :SignatureName then
+                signature = "[[User:#{@params.author}|#{@params.author}]]"
+                @signed_wiki[ast.index + @index_inc, ast.length] = signature
+                @index_inc += signature.length - ast.length
+            when :SignatureFull then
+                signature = "[[User:#{@params.author}|#{@params.author}]] #{@params.time}"
+                @signed_wiki[ast.index + @index_inc, ast.length] = signature
+                @index_inc += signature.length - ast.length
+          end
+        else
+            ast.contents
+        end
+    end
+
+end

data/lib/mediacloth/mediawikitemplatehandler.rb

@@ -0,0 +1,8 @@
+class MediaWikiTemplateHandler
+
+    def included_template(name, parameters)
+        serialized_parameters = "|" + parameters.join('|') if parameters and !parameters.empty?
+        "{{#{name}#{serialized_parameters}}}"
+    end
+
+end

data/lib/mediacloth/mediawikiwalker.rb

@@ -1,4 +1,5 @@
 require 'mediacloth/mediawikiast'
+require 'mediacloth/mediawikiparams'
 
 #Default walker to traverse the parse tree.
 #
@@ -14,6 +15,27 @@ class MediaWikiWalker
     def parse(ast)
         parse_wiki_ast(ast)
     end
+    
+    #Set generator's URL handler.
+    def link_handler=(handler)
+      @link_handler = handler
+    end
+    
+    #Returns's this generator URL handler. If no handler was set, returns the
+    #default handler.
+    def link_handler
+      @link_handler ||= MediaWikiLinkHandler.new
+    end
+
+    attr_writer :template_handler
+    def template_handler
+      @template_handler ||= MediaWikiTemplateHandler.new
+    end
+
+    attr_writer :params
+    def params
+        @params ||= MediaWikiParams.new
+    end
 
 protected
 
@@ -28,13 +50,19 @@ protected
             r = parse_preformatted(c) if c.class == PreformattedAST
             r = parse_section(c) if c.class == SectionAST
             r = parse_paragraph(c) if c.class == ParagraphAST
+            r = parse_paste(c) if c.class == PasteAST
             r = parse_link(c) if c.class == LinkAST
             r = parse_internal_link(c) if c.class == InternalLinkAST
             r = parse_resource_link(c) if c.class == ResourceLinkAST
+            r = parse_category_link(c) if c.class == CategoryLinkAST
             r = parse_internal_link_item(c) if c.class == InternalLinkItemAST
             r = parse_table(c) if c.class == TableAST
             r = parse_table_row(c) if c.class == TableRowAST
             r = parse_table_cell(c) if c.class == TableCellAST
+            r = parse_element(c) if c.class == ElementAST
+            r = parse_template(c) if c.class == TemplateAST
+            r = parse_category(c) if c.class == CategoryAST
+            r = parse_keyword(c) if c.class == KeywordAST
             r
         end
     end
@@ -44,6 +72,11 @@ protected
         parse_wiki_ast(ast)
     end
 
+    #Reimplement this
+    def parse_paste(ast)
+        parse_wiki_ast(ast)
+    end
+
     #Reimplement this
     def parse_formatted(ast)
         parse_wiki_ast(ast)
@@ -56,7 +89,10 @@ protected
     #Reimplement this
     def parse_list(ast)
         ast.children.map do |c|
-            parse_list_item(c) if c.class == ListItemAST
+            r = parse_list_item(c) if c.class == ListItemAST
+            r = parse_list_term(c) if c.class == ListTermAST
+            r = parse_list_definition(c) if c.class == ListDefinitionAST
+            r
         end
     end
 
@@ -65,8 +101,19 @@ protected
         parse_wiki_ast(ast)
     end
 
+    #Reimplement this
+    def parse_list_term(ast)
+        parse_wiki_ast(ast)
+    end
+
+    #Reimplement this
+    def parse_list_definition(ast)
+        parse_wiki_ast(ast)
+    end
+
     #Reimplement this
     def parse_preformatted(ast)
+        parse_wiki_ast(ast)
     end
 
     #Reimplement this
@@ -93,6 +140,13 @@ protected
         end
     end
 
+    #Reimplement this
+    def parse_category_link(ast)
+        ast.children.map do |c|
+            parse_internal_link_item(c) if c.class == InternalLinkItemAST
+        end
+    end
+
     #Reimplement this
     def parse_internal_link_item(ast)
         parse_wiki_ast(ast)
@@ -113,4 +167,21 @@ protected
         parse_wiki_ast(ast)
     end
 
+    #Reimplement this
+    def parse_element(ast)
+        parse_wiki_ast(ast)
+    end
+
+    #Reimplement this
+    def parse_template(ast)
+    end
+
+    #Reimplement this
+    def parse_category(ast)
+    end
+
+    #Reimplement this
+    def parse_keyword(ast)
+    end
+
 end