wptemplates 0.0.3

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+spec/examples_spec.rb
+test/tmp
+test/version_tmp
+tmp
+.irb_history
+README.html

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - "1.9.2"
+  - "1.9.3"
+  - jruby-19mode # JRuby in 1.9 mode
+  - rbx-19mode
+# uncomment this line if your project needs to run something other than `rake`:
+# script: bundle exec rspec spec

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in wptemplates.gemspec
+gemspec
+
+platforms :ruby_19, :rbx do
+  gem 'redcarpet', '~> 2'
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Bernhard Häussner
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,92 @@
+# Wptemplates
+
+[![Build Status](https://travis-ci.org/bxt/wptemplates.png?branch=master)](https://travis-ci.org/bxt/wptemplates)
+
+Gem for collecting template informations from mediawiki markup. 
+
+It will help you to extract useful machine-readable data from
+wikipedia articles, since there ist a lot of useful stuff
+encoded as templates.
+
+Currently only templates and links are parsed, all other markup is ignored.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'wptemplates', git: 'git://github.com/bxt/wptemplates.git'
+
+And then execute:
+
+    $ bundle
+
+The gem is currently not in the rubygems.org repository. 
+
+## Usage
+
+To parse a piece of markup simply call:
+
+<!-- EXAMPLES:INIT -->
+    ast = Wptemplates.parse("{{foo | bar | x = 3 }} baz [[bam (2003)|]]y")
+
+<!-- /EXAMPLES -->
+
+You will get an instance of Wptemplates::Soup which is an array of
+Wptemplates::Template, Wptemplates::Link and Wptemplates::Text. 
+You can explore the AST with these methods:
+
+<!-- EXAMPLES:intro -->
+    ast.templates.is_a?(Array) && ast.templates.length # => 1
+    ast.text # => " baz bamy"
+    ast[0].name # => :foo
+    ast[0].params[0].text # => " bar "
+    ast[0].params[:x].text # => "3"
+    ast.all_templates_of(:foo).map{|t| t.params[:x].text} # => ["3"]
+<!-- /EXAMPLES -->
+
+You can access the links via: 
+
+<!-- EXAMPLES:links -->
+    ast.links.length # => 1
+    ast.links[0].text # => "bamy"
+    ast.all_links.map{|l| l.link} # => ["Bam (2003)"]
+<!-- /EXAMPLES -->
+
+## Developing
+
+Here's some useful info if you want to improve/customize this gem. 
+
+### Getting Started
+
+Checkout the project, run `bundle` and then `rake` to see if the tests
+pass. Run `rake -T` to see the rake tasks. 
+
+### Markup
+
+MediaWiki markup is not trivial to parse and there might always
+be compatibility issues. There's a useful help page about 
+[templates][tmplh] and a [markup spec][mspec]. For links there
+is a page about [links][linkh] and about the [pipe trick][ptrkh]. 
+Also, there is a page with [link's BNF][lnbnf]. 
+
+### Known Issues
+
+* If you have images in your templates the pipes cause a new parameter
+* Namespaced links are not recognized
+* Templates in links are not recognized
+* Links contents are not htmldecoded
+* nowiki, pre and math blocks might cause problems
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+[tmplh]: http://en.wikipedia.org/wiki/Help:Template#Usage_syntax "English Wikipedia Template help page, syntax section"
+[mspec]: http://www.mediawiki.org/wiki/Markup_spec "MediaWiki Markup spec"
+[linkh]: http://en.wikipedia.org/wiki/Help:Link "English Wikipedia Link help page"
+[ptrkh]: http://en.wikipedia.org/wiki/Help:Pipe_trick "English Wikipedia Pipe trick help page"
+[lnbnf]: http://www.mediawiki.org/wiki/Markup_spec/BNF/Links "MediaWiki Link BNF"

data/README.rdoc

@@ -0,0 +1,8 @@
+= Wptemplates
+
+{<img src="https://travis-ci.org/bxt/wptemplates.png?branch=master" alt="Build Status" />}[https://travis-ci.org/bxt/wptemplates]
+
+The public entry method for this gem is Wptemplates.parse
+It will give you Wptemplates::Soup, Wptemplates::Link, 
+Wptemplates::Template and Wptemplates::Text elements, 
+which you can traverse and filter. 

data/Rakefile

@@ -0,0 +1,4 @@
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }
+
+task :default => :spec

data/lib/wptemplates.rb

@@ -0,0 +1,20 @@
+require "wptemplates/version"
+require "wptemplates/parser"
+require "wptemplates/preprocessor"
+
+module Wptemplates
+  
+  def self.parse text
+    parser.parse(text)
+    parser.parse(preprocessor.preprocess(text))
+  end
+  
+  def self.parser
+    @parser ||= Parser.new
+  end
+  
+  def self.preprocessor
+    @preprocessor ||= Preprocessor.new
+  end
+  
+end

data/lib/wptemplates/ast.rb

@@ -0,0 +1,91 @@
+module Wptemplates
+  
+  module Node
+    def templates
+      []
+    end
+    def all_templates
+      templates
+    end
+    def text
+      ""
+    end
+    def templates_of type
+      templates.select{|t| t.name==type}
+    end
+    def template_of type
+      templates_of(type).first
+    end
+    def all_templates_of type
+      all_templates.select{|t| t.name==type}
+    end
+    def deep_template_of type
+      all_templates_of(type).first
+    end
+    def links
+      []
+    end
+    def all_links
+      links
+    end
+  end
+  
+  class Soup < Array
+    include Node
+    def templates
+      map(&:templates).flatten(1)
+    end
+    def links
+      map(&:links).flatten(1)
+    end
+    def all_templates
+      map(&:all_templates).flatten(1)
+    end
+    def all_links
+      map(&:all_links).flatten(1)
+    end
+    def text
+      map(&:text).join('')
+    end
+  end
+  
+  class Template
+    include Node
+    attr_reader :name, :params
+    def initialize(name, params = {})
+      @name = name
+      @params = params
+    end
+    def templates
+      [self]
+    end
+    def all_templates
+      templates + @params.map{|_,v| v.templates }.flatten(1)
+    end
+    def all_links
+      links + @params.map{|_,v| v.all_links }.flatten(1)
+    end
+  end
+  
+  class Text
+    include Node
+    attr_reader :text
+    def initialize(text)
+      @text = text
+    end
+  end
+  
+  class Link
+    include Node
+    attr_reader :text, :link, :anchor
+    def initialize(text, link, anchor)
+      @text = text
+      @link = link
+      @anchor = anchor
+    end
+    def links
+      [self]
+    end
+  end
+  
+end

data/lib/wptemplates/parser.rb

@@ -0,0 +1,112 @@
+require 'strscan'
+require 'wptemplates/regexes'
+require 'wptemplates/utils'
+require 'wptemplates/ast'
+
+module Wptemplates
+  class Parser
+    include Regexes
+    include Utils
+    
+    def parse(text)
+      @input = StringScanner.new(text)
+      parse_main
+    end
+    
+  protected
+    
+    def parse_main in_template_parameter = false
+      output = Soup.new
+
+      while unit = parse_link || parse_template || parse_anything(in_template_parameter)
+        output << unit
+      end
+      
+      output << Text.new("") if output.empty?
+
+      output
+    end
+    
+    def parse_template
+      if @input.scan(a_doubleopenbrace)
+        template = Template.new parse_template_name, parse_template_parameters
+        @input.scan(a_doubleclosingbrace) or raise "unclosed template"
+        template
+      end
+    end
+    
+    def parse_anything in_template_parameter = false
+      if in_template_parameter
+        @input.scan(till_doublebrace_doubleopenbrackets_or_pipe) && Text.new(@input.matched)
+      else
+        @input.scan(till_doubleopenbrace_or_doubleopenbrackets) && Text.new(@input.matched)
+      end
+    end
+    
+    def parse_template_name
+      if @input.scan(till_doubleclosebrace_or_pipe)
+        symbolize(@input.matched)
+      end
+    end
+    
+    def parse_template_parameters
+      i = 0
+      h = {}
+      while parsed_named_template_parameter(h) || parse_numeric_template_parameter(h,i) do
+        i += 1
+      end
+      h
+    end
+    
+    def parsed_named_template_parameter(h)
+      if @input.scan(from_pipe_till_equals_no_doubleclosebrace_or_pipe)
+        key = symbolize(@input[1])
+        value = parse_main(true)
+        {l:0,r:-1}.each do |d,i|
+          value[i].text.send(:"#{d}strip!")
+          value.delete_at(i) if value[i].text.empty? && (value.length > 1)
+        end
+        h[key] = value
+      end
+    end
+    
+    def parse_numeric_template_parameter(h,i)
+      if @input.scan(a_pipe)
+        value = parse_main(true)
+        h[i] = value
+      end
+    end
+    
+    def parse_link
+      if @input.scan(a_link)
+        url, label, letters = (1..3).map {|i| @input[i]}
+        if label == ""
+          pipe_trick url, label, letters
+        else
+          link_new_with_normalize (label || url)+letters, url[until_hash], url[after_hash]
+        end
+      end
+    end
+    
+    def pipe_trick url, label, letters
+      if url["#"]
+        nil
+      elsif m = has_parens.match(url)
+        link_new_with_normalize(m[:no_parens]+letters, url, nil)
+      else
+        label = fixpoint(clone: true, start: url) do |u|
+          u[first_comma,:before][parens, :before]
+        end
+        link_new_with_normalize(label+letters, url, nil)
+      end
+    end
+    
+    def link_new_with_normalize text, link, anchor
+      text = normalize_linklabel(text)
+      link = normalize_link(link)
+      anchor = normalize_link(anchor, true) unless anchor.nil?
+      Link.new(text, link, anchor)
+    end
+    
+  end
+end

data/lib/wptemplates/preprocessor.rb

@@ -0,0 +1,17 @@
+
+module Wptemplates
+  class Preprocessor
+    
+    def preprocess(text)
+      strip_html_comments!(text)
+      text
+    end
+    
+  protected
+    
+    def strip_html_comments!(text)
+      text.gsub!(/<!--.*?-->/m,'')
+    end
+    
+  end
+end

data/lib/wptemplates/regexes.rb

@@ -0,0 +1,95 @@
+# encoding: UTF-8
+
+module Wptemplates
+  module Regexes
+    module_function
+    
+    def till_doublebrace_doubleopenbrackets_or_pipe
+      /(
+         [^{}\[|] # Unproblematic chars
+      |  { (?!{ ) # A lone open brace
+      |  } (?!} ) # A lone close brace
+      |  \[(?!\[) # A lone open bracket
+      |  ^\[\[    # Doubleopenbrackets at start
+      )+/x
+    end
+    
+    def till_doubleopenbrace_or_doubleopenbrackets
+      /(
+         [^{\[]   # Unproblematic chars
+      |  { (?!{ ) # A lone open brace
+      |  \[(?!\[) # A lone open bracket
+      |  ^\[\[    # Doubleopenbrackets at start
+      )+/x
+    end
+    
+    def till_doubleclosebrace_or_pipe
+      /(
+         [^|}]    # Unproblematic chars
+      |  } (?!} ) # A lone close brace
+      )+/x
+    end
+    
+    def from_pipe_till_equals_no_doubleclosebrace_or_pipe
+      /
+        \| # Pipe
+        ((
+         [^|=}]   # Unproblematic chars
+        |}(?!})   # A lone close brace
+        )*)
+        = # Equals
+      /x
+    end
+    
+    def a_pipe
+      /\|/
+    end
+    
+    def a_doubleopenbrace
+      /{{/
+    end
+    
+    def a_doubleclosingbrace
+      /}}/
+    end
+    
+    def a_link
+      /
+        \[\[
+          (?<link>
+            # ([% title-legal-chars])+
+            [%\ !"$&'()*,\-.\/0-9:;=?@A-Z\\^_`a-z~\u0080-\uFFFF+]+
+            # ("#" [# % title-legal-chars]+)?
+            ( \# [\#%\ !"$&'()*,\-.\/0-9:;=?@A-Z\\^_`a-z~\u0080-\uFFFF+]+ )?
+          )
+          (
+            # "|" LEGAL_ARTICLE_ENTITY*
+            \| (?<link-description>([^\]]|\](?!\]))*)
+          )?
+        \]\]
+        (?<extra_letters>\p{L}*)
+      /x
+    end
+    
+    def until_hash
+      /[^#]*/
+    end
+    
+    def after_hash
+      /(?<=#).*/
+    end
+    
+    def has_parens
+      /^(?<no_parens>.*?) *\(.*\) *$/
+    end
+    
+    def first_comma
+      /^(?<before>([^,]|,(?! ))*)(, |$)/
+    end
+    
+    def parens
+      /^(?<before>.*?)(\(.*\) *)?$/
+    end
+    
+  end
+end