wp2txt 0.9.5.1 → 1.0.0

data/bin/benchmark.rb

@@ -1,76 +0,0 @@
-#!/usr/bin/env ruby
-# -*- coding: utf-8 -*-
-
-$: << File.join(File.dirname(__FILE__))
-$: << File.join(File.dirname(__FILE__), '..', 'lib')
-
-require 'wp2txt'
-require 'wp2txt/utils'
-include Wp2txt
-require 'benchmark'
-
-data_dir = File.join(File.dirname(__FILE__), '..', "data")
-
-parent = Wp2txt::CmdProgbar.new
-input_file = File.join(data_dir, "testdata_ja.bz2")
-output_dir = data_dir
-tfile_size = 10
-num_threads = 1
-convert = true
-strip_tmarker = true
-
-Benchmark.bm do |x|
-  x.report do
-    wpconv = Wp2txt::Runner.new(parent, input_file, output_dir, tfile_size, num_threads, convert, strip_tmarker)
-    wpconv.extract_text do |article|
-      format_wiki!(article.title)
-      title = "[[#{article.title}]]\n"
-      convert_characters!(title)
-
-      contents = "\nCATEGORIES: "
-      contents += article.categories.join(", ")
-      contents += "\n\n"
-
-      article.elements.each do |e|
-        case e.first
-        when :mw_heading
-          format_wiki!(e.last)
-          line = e.last
-        when :mw_paragraph
-          format_wiki!(e.last)
-          line = e.last
-        when :mw_table, :mw_htable
-          format_wiki!(e.last)
-          line = e.last
-        when :mw_pre
-          line = e.last
-        when :mw_quote
-          format_wiki!(e.last)
-          line = e.last
-        when :mw_unordered, :mw_ordered, :mw_definition
-          format_wiki!(e.last)
-          line = e.last
-        when :mw_redirect
-          format_wiki!(e.last)
-          line = e.last
-          line += "\n\n"
-        else
-          next
-        end
-        contents << line
-      end
-      format_wiki!(contents)
-      convert_characters!(contents)
-
-      ##### cleanup #####
-      if /\A[\s　]*\z/m =~ contents
-        result = ""
-      else
-        result = title + "\n" + contents
-      end
-      result = result.gsub(/\[ref\]\s*\[\/ref\]/m){""}
-      result = result.gsub(/\n\n\n+/m){"\n\n"} + "\n"  
-    end
-  end
-end
-

data/lib/wp2txt/mw_api.rb

@@ -1,65 +0,0 @@
-#!/usr/bin/env ruby
-# -*- coding: utf-8 -*-
-
-$: << File.join(File.dirname(__FILE__))
-
-require 'uri'
-require 'net/http'
-require 'json'
-require 'utils'
-
-
-module Wp2txt
-
-  def post_request(uri_string, data={})
-    data = data.map{ |k, v| "#{k}=#{v}" }.join("&")
-    uri = URI.parse(uri_string)
-    uri.path = "/" if uri.path.empty?
-    http = Net::HTTP.new(uri.host)
-    return http.post(uri.path, data).body
-  end
-
-  def expand_template(uri, template, page)
-    text = URI.escape(template)
-    title = URI.escape(page)
-    data = {"action" => "expandtemplates",
-            "format" => "json",
-            "text"   => text,
-            "title"  => title}
-    jsn = post_request(uri, data)
-    hash = JSON.parse(jsn)
-    begin
-      result = hash["expandtemplates"]["*"]
-      result = special_chr(result)
-      return chrref_to_utf(result).gsub("{{", "&#123;&#123;").gsub("}}", "&#125;&#125;")
-    rescue => e      
-      puts "ERROR!"
-      p e
-      exit
-      template
-    end
-  end
-
-  def parse_wikitext(uri, wikitext, page)
-    text = URI.escape(wikitext)
-    title = URI.escape(page)
-    data = {"action" => "parse",
-            "format" => "json",
-            "text"   => text,
-            "title"  => title}
-    jsn = post_request(uri, data)
-    hash = JSON.parse(jsn)
-    begin
-      result = hash["parse"]["text"]["*"]
-      result = special_chr(result)
-      return chrref_to_utf(result).gsub("[[", "&#91;&#91;").gsub("]]", "&#93;&#93;")
-    rescue => e      
-      puts "ERROR!"
-      p e
-      exit
-      template
-    end
-  end
-  
-end
-

data/lib/wp2txt/progressbar.rb

@@ -1,305 +0,0 @@
-#!/usr/bin/env ruby
-# -*- coding: utf-8 -*-
-#
-# Ruby/ProgressBar - a text progress bar library
-#
-# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
-#     All rights reserved.
-#     This is free software with ABSOLUTELY NO WARRANTY.
-#
-# You can redistribute it and/or modify it under the terms
-# of Ruby's license.
-#
-
-module Wp2txt
-  class ProgressBar
- 
-    def initialize (title, total, out = STDERR)
-      @title = title
-      @total = total
-      @out = out
-      @terminal_width = 80
-      @bar_mark = "o"
-      @current = 0
-      @previous = 0
-      @finished_p = false
-      @start_time = Time.now
-      @previous_time = @start_time
-      @title_width = 14
-      @format = "%-#{@title_width}s %3d%% %s %s"
-      @format_arguments = [:title, :percentage, :bar, :stat]
-      clear
-      show
-    end
-    attr_reader   :title
-    attr_reader   :current
-    attr_reader   :total
-    attr_accessor :start_time
-
-    private
-    def fmt_bar
-      bar_width = do_percentage * @terminal_width / 100
-      sprintf("|%s%s|", 
-              @bar_mark * bar_width, 
-              " " *  (@terminal_width - bar_width))
-    end
-
-    def fmt_percentage
-      do_percentage
-    end
-
-    def fmt_stat
-      if @finished_p then elapsed else eta end
-    end
-
-    def fmt_stat_for_file_transfer
-      if @finished_p then 
-        sprintf("%s %s %s", bytes, transfer_rate, elapsed)
-      else 
-        sprintf("%s %s %s", bytes, transfer_rate, eta)
-      end
-    end
-
-    def fmt_title
-      @title[0,(@title_width - 1)] + ":"
-    end
-
-    def convert_bytes (bytes)
-      if bytes < 1024
-        sprintf("%6dB", bytes)
-      elsif bytes < 1024 * 1000 # 1000kb
-        sprintf("%5.1fKB", bytes.to_f / 1024)
-      elsif bytes < 1024 * 1024 * 1000  # 1000mb
-        sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
-      else
-        sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
-      end
-    end
-
-    def transfer_rate
-      bytes_per_second = @current.to_f / (Time.now - @start_time)
-      sprintf("%s/s", convert_bytes(bytes_per_second))
-    end
-
-    def bytes
-      convert_bytes(@current)
-    end
-
-    def format_time (t)
-      t = t.to_i
-      sec = t % 60
-      min  = (t / 60) % 60
-      hour = t / 3600
-      sprintf("%02d:%02d:%02d", hour, min, sec);
-    end
-
-    # ETA stands for Estimated Time of Arrival.
-    def eta
-      if @current == 0
-        "ETA:  --:--:--"
-      else
-        elapsed = Time.now - @start_time
-        eta = elapsed * @total / @current - elapsed;
-        sprintf("ETA:  %s", format_time(eta))
-      end
-    end
-
-    def elapsed
-      elapsed = Time.now - @start_time
-      sprintf("Time: %s", format_time(elapsed))
-    end
-  
-    def eol
-      if @finished_p then "\n" else "\r" end
-    end
-
-    def do_percentage
-      if @total.zero?
-        100
-      else
-        @current  * 100 / @total
-      end
-    end
-
-    def get_width
-      # FIXME: I don't know how portable it is.
-      default_width = 80
-      begin
-        tiocgwinsz = 0x5413
-        data = [0, 0, 0, 0].pack("SSSS")
-        if @out.ioctl(tiocgwinsz, data) >= 0 then
-          rows, cols, xpixels, ypixels = data.unpack("SSSS")
-          if cols >= 0 then cols else default_width end
-        else
-          default_width
-        end
-      rescue Exception
-        default_width
-      end
-    end
-
-    def show
-      arguments = @format_arguments.map {|method| 
-        method = sprintf("fmt_%s", method)
-        send(method)
-      }
-      line = sprintf(@format, *arguments)
-
-      width = get_width
-      if line.length == width - 1 
-        @out.print(line + eol)
-        @out.flush
-      elsif line.length >= width
-        @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
-        if @terminal_width == 0 then @out.print(line + eol) else show end
-      else # line.length < width - 1
-        @terminal_width += width - line.length + 1
-        show
-      end
-      @previous_time = Time.now
-    end
-
-    def show_if_needed
-      if @total.zero?
-        cur_percentage = 100
-        prev_percentage = 0
-      else
-        cur_percentage  = (@current  * 100 / @total).to_i
-        prev_percentage = (@previous * 100 / @total).to_i
-      end
-
-      # Use "!=" instead of ">" to support negative changes
-      if cur_percentage != prev_percentage || 
-          Time.now - @previous_time >= 1 || @finished_p
-        show
-      end
-    end
-
-    public
-    def clear
-      @out.print "\r"
-      @out.print(" " * (get_width - 1))
-      @out.print "\r"
-    end
-
-    def finish
-      @current = @total
-      @finished_p = true
-      show
-    end
-
-    def finished?
-      @finished_p
-    end
-
-    def file_transfer_mode
-      @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
-    end
-
-    def format= (format)
-      @format = format
-    end
-
-    def format_arguments= (arguments)
-      @format_arguments = arguments
-    end
-
-    def halt
-      @finished_p = true
-      show
-    end
-
-    def inc (step = 1)
-      @current += step
-      @current = @total if @current > @total
-      show_if_needed
-      @previous = @current
-    end
-
-    def set (count)
-      if count < 0 || count > @total
-        raise "invalid count: #{count} (total: #{@total})"
-      end
-      @current = count
-      show_if_needed
-      @previous = @current
-    end
-
-    def inspect
-      "#<ProgressBar:#{@current}/#{@total}>"
-    end
-  end
-
-  class ReversedProgressBar < ProgressBar
-    def do_percentage
-      100 - super
-    end
-  end
-
-  ##########################################
-
-  class NewProgressBar < ProgressBar
-
-    attr_accessor :nhits, :nsentences
-  
-    alias org_initialize initialize
-  
-    def initialize(filename, size)
-      @nhits = 0
-      @nsentences = 0
-      org_initialize(File.basename(filename), size)
-    end
-
-    alias org_fmt_stat fmt_stat
-  
-    def fmt_stat
-      org_fmt_stat # + " Hits: " + @nhits.to_s
-    end
-  end
-
-  class CmdProgbar
-
-    attr_accessor :pbar, :last_value
-  
-    def initialize
-      @last_value = 0
-      @pbar = nil
-    end
-  
-    def msg(str, i = nil)
-      case i
-      when 0
-        print str
-      else
-        puts str
-      end
-    end
-
-    def prg_update(value, elt, eta)
-      @elt = elt
-      @eta = eta
-      offset = value - @last_value
-      @pbar.inc(offset.to_i)
-      @last_value = value
-    end
-
-    def data_set(filename, linesize)
-      @pbar = NewProgressBar.new(filename, linesize)
-    end
-
-    def data_update(nhits, nsentences)
-      @pbar.nhits = nhits
-      @pbar.nsentences = nsentences
-    end  
-  
-    def before
-      initialize
-    end
-  
-    def after
-      @pbar.finish 
-      return true
-    end  
-
-  end
-end