wp2txt 0.4.1

data/lib/wp2txt/article.rb

@@ -0,0 +1,177 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+$: << File.join(File.dirname(__FILE__))
+
+require 'strscan'
+require 'utils'
+
+module Wp2txt
+
+  # possible element type, which could be later chosen to print or not to print
+  # :mw_heading
+  # :mw_htable
+  # :mw_quote
+  # :mw_unordered
+  # :mw_ordered
+  # :mw_definition
+  # :mw_pre
+  # :mw_paragraph
+  # :mw_comment
+  # :mw_math
+  # :mw_source
+  # :mw_inputbox
+  # :mw_template
+  # :mw_link
+  # :mw_summary
+  # :mw_blank
+  # :mw_redirect
+  
+  # an article contains elements, each of which is [TYPE, string]
+  class Article
+    
+    include Wp2txt
+    attr_accessor :elements, :title
+    
+    # class varialbes to save resource for generating regexps
+    # those with a trailing number 1 represent opening tag/markup
+    # those with a trailing number 2 represent closing tag/markup
+    # those without a trailing number contain both opening/closing tags/markups
+    
+    @@in_template_regex = Regexp.new('^\s*\{\{[^\}]+\}\}\s*$')
+    @@in_link_regex = Regexp.new('^\s*\[.*\]\s*$')
+        
+    @@in_inputbox_regex  = Regexp.new('<inputbox>.*?<\/inputbox>')
+    @@in_inputbox_regex1  = Regexp.new('<inputbox>')
+    @@in_inputbox_regex2  = Regexp.new('<\/inputbox>')
+    
+    @@in_source_regex  = Regexp.new('<source.*?>.*?<\/source>')
+    @@in_source_regex1  = Regexp.new('<source.*?>')
+    @@in_source_regex2  = Regexp.new('<\/source>')
+    
+    @@in_math_regex  = Regexp.new('<math.*?>.*?<\/math>')
+    @@in_math_regex1  = Regexp.new('<math.*?>')
+    @@in_math_regex2  = Regexp.new('<\/math>')
+    
+    @@in_heading_regex  = Regexp.new('^=+.*?=+$')
+    
+    @@in_html_table_regex = Regexp.new('<table.*?><\/table>')
+    @@in_html_table_regex1 = Regexp.new('<table\b')
+    @@in_html_table_regex2 = Regexp.new('<\/\s*table>')
+    
+    @@in_table_regex1 = Regexp.new('^\s*\{\|')
+    @@in_table_regex2 = Regexp.new('^\|\}.*?$')
+    
+    @@in_unordered_regex  = Regexp.new('^\*')
+    @@in_ordered_regex    = Regexp.new('^\#')
+    @@in_pre_regex = Regexp.new('^ ')
+    @@in_definition_regex  = Regexp.new('^[\;\:]')    
+    
+    @@blank_line_regex = Regexp.new('^\s*$')
+
+    @@redirect_regex = Regexp.new('#(?:REDIRECT|転送)\s+\[\[(.+)\]\]', Regexp::IGNORECASE)
+
+    def initialize(text, title = "", strip_tmarker = false)
+      @title = title.strip
+      @strip_tmarker = strip_tmarker
+      parse text
+    end
+    
+    def create_element(tp, text)
+      [tp, text]
+    end
+  
+    def parse(source)
+      @elements = []
+      mode = nil
+      open_stack  = []
+      close_stack = []
+      source.each_line do |line|
+
+        case mode
+        when :mw_table
+          if @@in_table_regex2 =~ line
+            mode = nil
+          end
+          @elements.last.last << line
+          next          
+        when :mw_inputbox
+          if @@in_inputbox_regex2 =~ line
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        when :mw_source
+          if @@in_source_regex2 =~ line
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        when :mw_math
+          if @@in_math_regex2 =~ line
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        when :mw_htable
+          if @@in_html_table_regex2 =~ line
+            mode = nil
+          end
+          @elements.last.last << line
+          next
+        end
+
+        case line
+        when @@blank_line_regex
+          @elements << create_element(:mw_blank, "\n")      
+        when @@redirect_regex
+          @elements << create_element(:mw_redirect, line)
+        when @@in_template_regex
+          @elements << create_element(:mw_template, line)
+        when @@in_heading_regex
+          @elements << create_element(:mw_heading, "\n" + line + "\n")
+        when @@in_inputbox_regex
+          @elements << create_element(:mw_inputbox, line)
+        when @@in_inputbox_regex1
+          mode = :mw_inputbox 
+          @elements << create_element(:mw_inputbox, line)
+        when @@in_source_regex
+        @elements << create_element(:mw_source, line)
+        when @@in_source_regex1
+          mode = :mw_source
+          @elements << create_element(:mw_source, line)
+        when @@in_math_regex
+          @elements << create_element(:mw_math, line)
+        when @@in_math_regex1
+          mode = :mw_math
+          @elements << create_element(:mw_math, line)
+        when @@in_html_table_regex
+          @elements << create_element(:mw_htable, line)
+        when @@in_html_table_regex1
+          mode = :mw_htable
+          @elements << create_element(:mw_htable, line)
+        when @@in_table_regex1
+          mode = :mw_table
+          @elements << create_element(:mw_table, line)
+        when @@in_unordered_regex
+          line = line.sub(/\A[\*\#\;\:\ ]+/, "") if @strip_tmarker          
+          @elements << create_element(:mw_unordered, line)
+        when @@in_ordered_regex
+          line = line.sub(/\A[\*\#\;\:\ ]+/, "") if @strip_tmarker          
+          @elements << create_element(:mw_ordered, line)
+        when @@in_pre_regex
+          line = line.sub(/\A\^\ /, "") if @strip_tmarker          
+          @elements << create_element(:mw_pre, line)
+        when @@in_definition_regex
+          line = line.sub(/\A[\;\:\ ]+/, "") if @strip_tmarker
+          @elements << create_element(:mw_definition, line)
+        when @@in_link_regex
+          @elements << create_element(:mw_link, line)
+        else 
+          @elements << create_element(:mw_paragraph, line)
+        end
+      end
+      @elements
+    end
+  end
+end

data/lib/wp2txt/mw_api.rb

@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+$: << File.join(File.dirname(__FILE__))
+
+require 'uri'
+require 'net/http'
+require 'json'
+require 'utils'
+
+
+module Wp2txt
+
+  def post_request(uri_string, data={})
+    data = data.map{ |k, v| "#{k}=#{v}" }.join("&")
+    uri = URI.parse(uri_string)
+    uri.path = "/" if uri.path.empty?
+    http = Net::HTTP.new(uri.host)
+    return http.post(uri.path, data).body
+  end
+
+  def expand_template(uri, template, page)
+    text = URI.escape(template)
+    title = URI.escape(page)
+    data = {"action" => "expandtemplates",
+            "format" => "json",
+            "text"   => text,
+            "title"  => title}
+    jsn = post_request(uri, data)
+    hash = JSON.parse(jsn)
+    begin
+      result = hash["expandtemplates"]["*"]
+      result = special_chr(result)
+      return chrref_to_utf(result).gsub("{{", "&#123;&#123;").gsub("}}", "&#125;&#125;")
+    rescue => e      
+      puts "ERROR!"
+      p e
+      exit
+      template
+    end
+  end
+
+  def parse_wikitext(uri, wikitext, page)
+    text = URI.escape(wikitext)
+    title = URI.escape(page)
+    data = {"action" => "parse",
+            "format" => "json",
+            "text"   => text,
+            "title"  => title}
+    jsn = post_request(uri, data)
+    hash = JSON.parse(jsn)
+    begin
+      result = hash["parse"]["text"]["*"]
+      result = special_chr(result)
+      return chrref_to_utf(result).gsub("[[", "&#91;&#91;").gsub("]]", "&#93;&#93;")
+    rescue => e      
+      puts "ERROR!"
+      p e
+      exit
+      template
+    end
+  end
+  
+end
+

data/lib/wp2txt/progressbar.rb

@@ -0,0 +1,305 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+module Wp2txt
+  class ProgressBar
+ 
+    def initialize (title, total, out = STDERR)
+      @title = title
+      @total = total
+      @out = out
+      @terminal_width = 80
+      @bar_mark = "o"
+      @current = 0
+      @previous = 0
+      @finished_p = false
+      @start_time = Time.now
+      @previous_time = @start_time
+      @title_width = 14
+      @format = "%-#{@title_width}s %3d%% %s %s"
+      @format_arguments = [:title, :percentage, :bar, :stat]
+      clear
+      show
+    end
+    attr_reader   :title
+    attr_reader   :current
+    attr_reader   :total
+    attr_accessor :start_time
+
+    private
+    def fmt_bar
+      bar_width = do_percentage * @terminal_width / 100
+      sprintf("|%s%s|", 
+              @bar_mark * bar_width, 
+              " " *  (@terminal_width - bar_width))
+    end
+
+    def fmt_percentage
+      do_percentage
+    end
+
+    def fmt_stat
+      if @finished_p then elapsed else eta end
+    end
+
+    def fmt_stat_for_file_transfer
+      if @finished_p then 
+        sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+      else 
+        sprintf("%s %s %s", bytes, transfer_rate, eta)
+      end
+    end
+
+    def fmt_title
+      @title[0,(@title_width - 1)] + ":"
+    end
+
+    def convert_bytes (bytes)
+      if bytes < 1024
+        sprintf("%6dB", bytes)
+      elsif bytes < 1024 * 1000 # 1000kb
+        sprintf("%5.1fKB", bytes.to_f / 1024)
+      elsif bytes < 1024 * 1024 * 1000  # 1000mb
+        sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+      else
+        sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+      end
+    end
+
+    def transfer_rate
+      bytes_per_second = @current.to_f / (Time.now - @start_time)
+      sprintf("%s/s", convert_bytes(bytes_per_second))
+    end
+
+    def bytes
+      convert_bytes(@current)
+    end
+
+    def format_time (t)
+      t = t.to_i
+      sec = t % 60
+      min  = (t / 60) % 60
+      hour = t / 3600
+      sprintf("%02d:%02d:%02d", hour, min, sec);
+    end
+
+    # ETA stands for Estimated Time of Arrival.
+    def eta
+      if @current == 0
+        "ETA:  --:--:--"
+      else
+        elapsed = Time.now - @start_time
+        eta = elapsed * @total / @current - elapsed;
+        sprintf("ETA:  %s", format_time(eta))
+      end
+    end
+
+    def elapsed
+      elapsed = Time.now - @start_time
+      sprintf("Time: %s", format_time(elapsed))
+    end
+  
+    def eol
+      if @finished_p then "\n" else "\r" end
+    end
+
+    def do_percentage
+      if @total.zero?
+        100
+      else
+        @current  * 100 / @total
+      end
+    end
+
+    def get_width
+      # FIXME: I don't know how portable it is.
+      default_width = 80
+      begin
+        tiocgwinsz = 0x5413
+        data = [0, 0, 0, 0].pack("SSSS")
+        if @out.ioctl(tiocgwinsz, data) >= 0 then
+          rows, cols, xpixels, ypixels = data.unpack("SSSS")
+          if cols >= 0 then cols else default_width end
+        else
+          default_width
+        end
+      rescue Exception
+        default_width
+      end
+    end
+
+    def show
+      arguments = @format_arguments.map {|method| 
+        method = sprintf("fmt_%s", method)
+        send(method)
+      }
+      line = sprintf(@format, *arguments)
+
+      width = get_width
+      if line.length == width - 1 
+        @out.print(line + eol)
+        @out.flush
+      elsif line.length >= width
+        @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+        if @terminal_width == 0 then @out.print(line + eol) else show end
+      else # line.length < width - 1
+        @terminal_width += width - line.length + 1
+        show
+      end
+      @previous_time = Time.now
+    end
+
+    def show_if_needed
+      if @total.zero?
+        cur_percentage = 100
+        prev_percentage = 0
+      else
+        cur_percentage  = (@current  * 100 / @total).to_i
+        prev_percentage = (@previous * 100 / @total).to_i
+      end
+
+      # Use "!=" instead of ">" to support negative changes
+      if cur_percentage != prev_percentage || 
+          Time.now - @previous_time >= 1 || @finished_p
+        show
+      end
+    end
+
+    public
+    def clear
+      @out.print "\r"
+      @out.print(" " * (get_width - 1))
+      @out.print "\r"
+    end
+
+    def finish
+      @current = @total
+      @finished_p = true
+      show
+    end
+
+    def finished?
+      @finished_p
+    end
+
+    def file_transfer_mode
+      @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+    end
+
+    def format= (format)
+      @format = format
+    end
+
+    def format_arguments= (arguments)
+      @format_arguments = arguments
+    end
+
+    def halt
+      @finished_p = true
+      show
+    end
+
+    def inc (step = 1)
+      @current += step
+      @current = @total if @current > @total
+      show_if_needed
+      @previous = @current
+    end
+
+    def set (count)
+      if count < 0 || count > @total
+        raise "invalid count: #{count} (total: #{@total})"
+      end
+      @current = count
+      show_if_needed
+      @previous = @current
+    end
+
+    def inspect
+      "#<ProgressBar:#{@current}/#{@total}>"
+    end
+  end
+
+  class ReversedProgressBar < ProgressBar
+    def do_percentage
+      100 - super
+    end
+  end
+
+  ##########################################
+
+  class NewProgressBar < ProgressBar
+
+    attr_accessor :nhits, :nsentences
+  
+    alias org_initialize initialize
+  
+    def initialize(filename, size)
+      @nhits = 0
+      @nsentences = 0
+      org_initialize(File.basename(filename), size)
+    end
+
+    alias org_fmt_stat fmt_stat
+  
+    def fmt_stat
+      org_fmt_stat # + " Hits: " + @nhits.to_s
+    end
+  end
+
+  class CmdProgbar
+
+    attr_accessor :pbar, :last_value
+  
+    def initialize
+      @last_value = 0
+      @pbar = nil
+    end
+  
+    def msg(str, i = nil)
+      case i
+      when 0
+        print str
+      else
+        puts str
+      end
+    end
+
+    def prg_update(value, elt, eta)
+      @elt = elt
+      @eta = eta
+      offset = value - @last_value
+      @pbar.inc(offset.to_i)
+      @last_value = value
+    end
+
+    def data_set(filename, linesize)
+      @pbar = NewProgressBar.new(filename, linesize)
+    end
+
+    def data_update(nhits, nsentences)
+      @pbar.nhits = nhits
+      @pbar.nsentences = nsentences
+    end  
+  
+    def before
+      initialize
+    end
+  
+    def after
+      @pbar.finish 
+      return true
+    end  
+
+  end
+end