wikipedia_parser 1.0.3 → 1.0.5

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZmE5MjA2NGQ4NTg5Zjc2OGQ5NzYwNWRjZGU3ZTM5MjM4MDA0ZDI0ZQ==
+    YzAxOTllY2I0ZmY0NTI1ZjFiMDUzN2EzNmJiMzVmYzljZDE1MDY3Nw==
   data.tar.gz: !binary |-
-    YjVjMTUzZjI5ZmQzNDU4YzM2ZTcwMmEzYzlhMmViOTkwNzg0ZTJlOQ==
+    MTFiOTI2NzNiOWI0MDEyOTkwODllZTQwYmNhYjAzN2Q0YWJhZWIxMw==
 SHA512:
   metadata.gz: !binary |-
-    N2JkYTkxMjc4MmEwN2VhMjU5ZTZlMjc4ZTJkNzdlMmJhYjUxN2ZiMzc4NTVj
-    ZDczMWUxNTEwMWIyY2Q1NjVjYzZjMGEzYTY0MmE5NDE3MTVkNTU0OGY3NDJi
-    OTViMzRkN2QzMWU2YmE0NzY2ZWM4M2I2NzFjYTQ3N2QzZjY3YWQ=
+    ODI1YjBmMGIwOTYyZDlmNDYwMmNjMTNjOTgyZGMyMTY5ZTdhM2ZmMGEyZTY4
+    NjU1YTVkNDU4OTRkOGMzMzk1ZDA5NmFjNTgwYjIzNmZiMmY0Y2Y1NDk5YWU1
+    YzVkZjllZGU0NDE1MDU0NGI3NGJkZjI5MTU5ZWYyOGNmZWQzZjc=
   data.tar.gz: !binary |-
-    ZGM0NGY4Y2RkZGYzYTk1MGRlYjY4OTRjNzFiNGEzMDQwMmVkMjY0M2U1OTJh
-    NDVjNTlhMDM1MDQwYzZhYzYyNTA2ZGI0NzE4MTJhOGUzZmMzODBmZGQ4YzVi
-    MTc5MTk4MTE5OWQ4OTM3ODU5ZjZlYWRjMWIwZWQyMjMyNzYwMjA=
+    Y2M5YzYwNGNhODA4MWY5YzdjMDc4Y2FjODJiMTgyYjMxZDAwYTE0NGU1NTlm
+    Y2IyMTkyMzQ2NDYyNDllNzY0ZmYxOWE3ODRhMzk3M2M1ZmRmOTMxZGFkY2Zh
+    YjdjYzM3ZmI3NWI2MDU2NDdlZjE4YTVlZGM2ZGEyMTM5ZTFlMjM=

data/lib/wikiParser.rb

@@ -0,0 +1,68 @@
+# coding: utf-8
+require 'nokogiri'
+require 'fileutils'
+require(File.dirname(__FILE__)+'/wikiParserPage.rb')
+
+# Parses a Wikipedia dump and extracts internal links, content, and page type.
+class WikiParser
+
+	# path to the Wikipedia dump.
+	attr_reader :path
+
+	# Convert the opened path to a dump to an enumerator of {WikiParser::Page}
+	# @return [Enumerator<Nokogiri::XML::Node>] the enumerator.
+	def parse
+		@xml_file = File.open(@path)
+		@file = Nokogiri::XML::Reader((@path.match(/.+\.bz2/) ? (require 'bzip2';Bzip2::Reader.open(@path)) : @xml_file), nil, 'utf-8', Nokogiri::XML::ParseOptions::NOERROR)
+		@reader = @file.to_enum
+	end
+
+	# Convert the opened path to a dump to an enumerator of {WikiParser::Page}
+	# @param opts [Hash] the parameters to parse a wikipedia page.
+	# @option opts [String] :path The path to the Wikipedia dump in .xml or .bz2 format.
+	# @return [Enumerator<Nokogiri::XML::Node>] the enumerator.
+	def initialize (opts = {})
+		@file, new_path = nil, opts[:path]
+		if File.exists? new_path and !File.directory? new_path
+			@path = new_path
+			parse
+		else
+			raise ArgumentError.new "Cannot open file. Check path please."
+		end
+	end
+
+	# Closes the file reader.
+	def close; @xml_file.close if @xml_file; end
+
+	# Skips a {WikiParser::Page} in the enumeration
+	def skip
+		begin
+			node = @reader.next
+			if node.name == "page" and node.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT
+			else
+				skip
+			end
+		rescue StopIteration
+			nil
+		end
+	end
+
+	# Reads the next node in the xml tree and returns it as a {WikiParser#::Page} if it exists.
+	# @return [WikiParser::Page, NilClass] A page if found.
+	# @param opts [Hash] the parameters to instantiate a page.
+	# @option opts [String] :until A node-name stopping point for the parsing. (Useful for not parsing an entire page until some property is checked.)
+	# @see Page#finish_processing
+	def get_next_page(opts={})
+		begin
+			node = @reader.next
+			if node.name == "page" and node.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT
+				xml = Nokogiri::XML::parse("<page>"+node.inner_xml+"</page>").first_element_child
+				return WikiParser::Page.new({:node => xml}.merge(opts))
+			else
+				get_next_page(opts)
+			end
+		rescue StopIteration, NoMethodError
+			nil
+		end
+	end
+end

data/lib/wikiParserPage.rb

@@ -0,0 +1,98 @@
+class WikiParser
+
+	# A Wikipedia article page object.
+	class Page
+
+		# The Wikipedia namespaces for all special pages {WikiParser::Page#special_page}, {#page_type}.
+		Namespaces = %w(WP Help Talk User Template Wikipedia File Book Portal TimedText Module MediaWiki Special Media Category)
+		# Title of the Wikipedia article.
+		attr_reader :title
+		# The Wikipedia id of the article.
+		attr_reader :id
+		attr_reader :internal_links
+		# the content of the Wikipedia article
+		attr_reader :article
+		# is this page a redirection page?
+		attr_reader :redirect
+		# the title of the page this article redirects to.
+		attr_reader :redirect_title
+		# the wikipedia namespace for this page
+		attr_reader :page_type
+		# is this page `special`? Is it in the {Namespaces}?
+		attr_reader :special_page
+		attr_reader :disambiguation_page
+
+		# Create a new article page from an XML node.
+		# @param opts [Hash] the parameters to instantiate a page.
+		# @option opts [Nokogiri::XML::Node] :node the {http://rubydoc.info/gems/nokogiri/frames Nokogiri::XML::Node} containing the article.
+		# @option opts [Fixnum] :from the index from which to resume parsing among the nodes.
+		# @option opts [String] :until A node-name stopping point for the parsing.
+		def initialize (opts={})
+			@title    = @article      = @redirect_title      = ""
+			@redirect = @special_page = @disambiguation_page = false
+			@internal_links, @page_type = [], nil
+			return unless !opts[:node].nil?
+			process_node opts
+			trigs = article_to_internal_links(@article)
+			@internal_links = trigs
+		end
+
+		def process_node(opts={})
+			opts[:node].element_children.each_with_index do |node,k|
+				if opts[:from] and k < opts[:from] then next end
+				case node.name
+				when 'id'
+					@id    = node.content
+				when 'title'
+					@title = node.content
+					if @title.match(/(#{Namespaces.join("|")}):.+/i) then @special_page = true and @page_type = $1 end
+					if @title.match(/.+ \(disambiguation\)/i)      then @disambiguation_page = true end
+				when 'redirect'
+					@redirect = true
+					@redirect_title = node["title"]
+				when 'revision'
+					node.element_children.each do |rev_node|
+						if rev_node.name == 'text'
+							@article = rev_node.content
+						end
+					end
+				end
+				if opts[:until] and opts[:until] == node.name
+					@stop_index = k
+					break
+				end
+			end
+		end
+
+		# Extracts internals links from a wikipedia article into an array of `uri`s and `title`s, starting
+		# from the stopping point given to the parser earlier.
+		# @return [WikiParser::Page] the parser.
+		def finish_processing
+			@stop_index||= 0
+			process_node :node => @node, :from => @stop_index
+			self
+		end
+
+		# Extracts internals links from a wikipedia article into an array of `uri`s and `title`s:
+		# @param article [String] the article content to extract links from.
+		# @return [Array<Hash>] the internal links in hash form.
+		def article_to_internal_links article
+			links = []
+			matches = article.scan(/\[\[(?<name>[^\]\|:]+)(?<trigger>\|[^\]]+)?\]\]/)
+			if matches
+				matches.each do |match|
+					name_match = match[0].strip.chomp.match(/^(?<name>[^#]+)(?<hashtag>#.+)?/)
+					link_match = match[1] ? match[1].strip.chomp.match(/^\|[\t\n\s\/]*(?<name>[^#]+)(?<hashtag>#.+)?/) : name_match
+					if name_match
+						name_match = name_match[:name].gsub('_', ' ')
+						link_match = link_match ? link_match[:name] : name_match
+						links << {:uri => name_match, :title => link_match}
+					end
+				end
+			end
+			links
+		end
+
+		private :process_node
+	end
+end

data/lib/wikipedia_parser.rb

@@ -0,0 +1 @@
+require(File.dirname(__FILE__)+'/wikiParser.rb')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wikipedia_parser
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.5
 platform: ruby
 authors:
 - Jonathan Raiman
@@ -58,7 +58,10 @@ email: jraiman@mit.edu
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- lib/wikiParser.rb
+- lib/wikiParserPage.rb
+- lib/wikipedia_parser.rb
 homepage: http://github.org/JonathanRaiman/wikipedia_parser
 licenses:
 - MIT