wp2txt 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c05af7e5c72b073f18b53eca8619212f0928aaa1
-  data.tar.gz: 02dc116458041b096fd811b3fec6ffb6e6ff3ee7
+  metadata.gz: bcfd6986e262e455c100d664583b099d57ff4428
+  data.tar.gz: 68c922b43951b7f326b0136681981a166208d0a9
 SHA512:
-  metadata.gz: 26479a43e8e3ccebfb6578562d62b7b6d2234e924838774a11d143c8fad7e1952db0278526d202a77aad680926b96baa25c8da9e0c23f916d95ccf614d5e82a3
-  data.tar.gz: 8959c9b51efb18386cf556c67439332fc43723d8de8abfc014745d0d0ad1f89ba56b6222b6f8d2a33748343e2dbef93752ee104c8a54c57e01fe21bee7e2f892
+  metadata.gz: 294e0f8e1d2b37534ad885c617cfbd72ad72144dca6fb01231f6e2cf691a86bf58690f5dd1b2b410f8ee23eb3c74fa3f40e4ca8bbf3f3921ea78295783da5f2e
+  data.tar.gz: 71a1b8feca5c3067ff534f0239c4c937485ff3ed8c0a6de793be0b71befd440cb5b1c2c465dc6684c558a53209ab6481dd2e60edfe7fb7cbdd9c6f07416efd24

data/bin/benchmark.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+$: << File.join(File.dirname(__FILE__))
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'wp2txt'
+require 'wp2txt/utils'
+include Wp2txt
+require 'benchmark'
+
+data_dir = File.join(File.dirname(__FILE__), '..', "data")
+
+parent = Wp2txt::CmdProgbar.new
+input_file = File.join(data_dir, "testdata.bz2")
+output_dir = data_dir
+tfile_size = 10
+convert = true
+strip_tmarker = true
+
+
+
+Benchmark.bm do |x|
+  x.report do
+    wpconv = Wp2txt::Runner.new(parent, input_file, output_dir, tfile_size, convert, strip_tmarker)
+    wpconv.extract_text do |article|
+      title = format_wiki article.title
+      title = "[[#{title}]]\n"
+
+        contents = "\nCATEGORIES: "
+        contents += article.categories.join(", ")
+        contents += "\n\n"
+
+      article.elements.each do |e|
+        case e.first
+        when :mw_heading
+          line = format_wiki(e.last)
+        when :mw_paragraph
+          line = format_wiki(e.last)
+        when :mw_table, :mw_htable
+          line = format_wiki(e.last)
+        when :mw_pre
+          line = e.last
+        when :mw_quote
+          line = format_wiki(e.last)
+        when :mw_unordered, :mw_ordered, :mw_definition
+          line = format_wiki(e.last)
+        when :mw_redirect
+          line = format_wiki(e.last)
+          line += "\n\n"
+        else
+          next
+        end
+        contents += line
+        contents = remove_templates(contents)
+      end
+    
+      ##### cleanup #####
+      if /\A\s*\z/m =~ contents
+        result = ""
+      else
+        result = title + "\n" + contents
+      end
+      result = result.gsub(/\[ref\]\s*\[\/ref\]/m){""}
+      result = result.gsub(/\n\n\n+/m){"\n\n"} + "\n"  
+    end
+  end
+end
+

data/bin/wp2txt

@@ -45,7 +45,7 @@ output_dir = opts[:output_dir]
 tfile_size = opts[:file_size]
 convert = opts[:convert]
 strip_tmarker = opts[:marker] ? false : true
-opt_array = [:title_off, :list, :heading, :table, :template, :redirect]
+opt_array = [:title, :list, :heading, :table, :template, :redirect]
 config = {}
 opt_array.each do |opt|
   config[opt] = opts[opt]

data/lib/wp2txt/version.rb

@@ -1,3 +1,3 @@
 module Wp2txt
-  VERSION = "0.6.0"
+  VERSION = "0.6.1"
 end

data/wp2txt.gemspec

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+ # -*- coding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
 require "wp2txt/version"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wp2txt
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Yoichiro Hasebe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-04 00:00:00.000000000 Z
+date: 2014-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -99,6 +99,7 @@ description: WP2TXT extracts plain text data from Wikipedia dump file (encoded i
 email:
 - yohasebe@gmail.com
 executables:
+- benchmark.rb
 - wp2txt
 extensions: []
 extra_rdoc_files: []
@@ -108,6 +109,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/benchmark.rb
 - bin/wp2txt
 - data/testdata.bz2
 - lib/wp2txt.rb
@@ -138,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: wp2txt
-rubygems_version: 2.4.1
+rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: Wikipedia dump to text converter