mediawiki-keiki 1.0.0

data/Gemfile

@@ -0,0 +1,16 @@
+source 'https://rubygems.org'
+
+gemspec
+
+# gem 'httparty'
+# gem 'wikicloth'
+# gem 'nokogiri'
+
+# group :test do
+# 	gem 'webmock'
+# 	gem 'vcr'
+# 	gem 'turn'
+# 	gem 'rake'
+# 	gem 'pry'
+# 	gem 'minitest-reporters'
+# end

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2010-2014 Google, Inc. http://angularjs.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,14 @@
+# MediaWiki-Keiki
+
+A Ruby API client for the [MediaWiki API](http://www.mediawiki.org/wiki/API:Main_page)
+
+## Features
+
+
+## Links
+
+
+## Installation
+
+
+## Usage

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+ 
+Rake::TestTask.new do |t|
+  t.test_files = FileList['spec/lib/mediawiki-keiki/*_spec.rb']
+  t.verbose = true
+end
+ 
+task :default => :test

data/lib/mediawiki-keiki/page.rb

@@ -0,0 +1,48 @@
+module MediaWiki
+
+	class Page
+
+		attr_reader :page_data
+
+		def initialize(hash)
+			@page_data = hash
+		end
+
+		# Modify method_missing to return results if a matching key can be found within the page hash
+		def method_missing(name, *args, &block)
+			@page_data.has_key?(name.to_s) ? @page_data[name.to_s] : super
+		end
+
+		# Gets the content by looking inside the 'revisions' key in the page hash
+		def content
+			revisions[0]["*"]
+		end
+
+		# Uses the WikiCloth gem to convert the content from WikiMarkup to HTML
+		def to_html
+			WikiCloth::Parser.new( :data => content ).to_html
+		end
+
+		# Converts the content to plain text
+		def to_text
+			Nokogiri::HTML(to_html).text
+		end
+
+		# Returns a short summary that is at least 140 characters long
+		def summary
+			text_array = to_text.split("\n")
+			text = text_array[0]
+			i = 1
+
+			while text.length <= 140 && i < text_array.length
+				text << "\n" + text_array[i]
+				i += 1
+			end
+
+			text
+
+		end
+
+	end
+
+end

data/lib/mediawiki-keiki/query.rb

@@ -0,0 +1,56 @@
+module MediaWiki
+
+	class Query
+
+		attr_accessor :query, :query_result
+
+		include HTTParty
+
+		# Sets the User-Agent header for the HTTP GET request
+		headers 'User-Agent' => 'crawler/1.0 (https://github.com/mkulumadzi/crawler)'
+
+		# Sets the base-uri to the Wikimedia API endpoint
+		base_uri 'https://en.wikipedia.org'
+
+
+		def initialize(query)
+
+			# The WikiMedia API requires that requests be limited to 50 sites or less
+			raise ArgumentError, "Query exceeds WikiMedia maximum number of sites (50)" unless query.count("|") < 50
+
+			@query = query
+			@page_hash = Hash.new
+
+		end
+
+		# Unless force is true, uses existing query_result if it has already been cached
+		def query_result(force = false)
+			force ? @query_result = get_query_result : @query_result ||= get_query_result
+		end
+
+		# Returns a hash filled with Pages 
+		def pages
+
+			# Captures the original query and sorts it, for using as keys with the hash
+			key_array = @query.split('|').sort
+			i = 0
+
+			# Creates a hash, using the original query as the keys and new Site objects as the values
+			query_result["query"]["pages"].each do |key, value|
+				@page_hash[key_array[i]] = MediaWiki::Page.new(value)
+				i += 1
+			end
+
+			@page_hash
+		end
+
+		private
+
+		# Private method that gets called if the query_result has not been retrieved yet
+		def get_query_result
+			self.class.get URI.encode("/w/api.php?continue=&format=json&action=query&titles=#{@query}&prop=revisions&rvprop=content&redirects")
+		end
+
+	end
+
+end

data/lib/mediawiki-keiki.rb

@@ -0,0 +1,7 @@
+require 'httparty'
+require 'wikicloth'
+require 'nokogiri'
+
+Dir[File.dirname(__FILE__) + '/mediawiki-keiki/*.rb'].each do |file|
+  require file
+end

data/spec/lib/mediawiki-keiki/page_spec.rb

@@ -0,0 +1,84 @@
+require_relative '../../spec_helper'
+
+describe MediaWiki::Page do
+
+	describe "Retrieve a page" do
+
+		let(:page) { MediaWiki::Query.new('foo').pages['foo'] }
+
+		before do
+			VCR.insert_cassette 'page', :record => :new_episodes
+		end
+
+		after do
+			VCR.eject_cassette
+		end
+
+		describe "get valid page" do
+
+			it "must get a valid page from WikiQuery pages" do
+				page.must_be_instance_of MediaWiki::Page
+			end
+
+			it "must store the page data as a hash" do
+				page.page_data.must_be_instance_of Hash
+			end
+
+			it "must return the page title" do
+				page.title.must_equal "Foobar"
+			end
+
+			it "must raise method missing if attribute is not present" do
+				lambda { page.foo_attribute }.must_raise NoMethodError
+			end
+
+			it "must not point to a redirect" do
+				/REDIRECT/.match(page.content).must_equal nil
+			end
+
+			it "must return the page content" do
+				page.content.must_be_instance_of String
+			end
+
+			it "must parse the page content as html" do
+				page.to_html.include?('<a href=').must_equal true
+			end
+
+			it "must parse the page content to plain text" do
+				page.to_text.include?('<a href=').must_equal false
+			end
+
+			it "must return a summary of the text" do
+				page.summary.must_be_instance_of String
+			end
+
+			it "must return at least 140 characters in the short summary" do
+				assert_operator(page.summary.length, :>=, 140)
+			end
+
+		end
+
+		describe "missing page" do
+
+			let(:page) { MediaWiki::Query.new('Foolicious').pages['Foolicious'] }
+
+			it "must be flagged as missing" do
+				page.missing.must_equal ""
+			end
+
+		end
+
+		describe "get invalid page" do
+
+			let(:page) { MediaWiki::Query.new('Talk:').pages['Talk:'] }
+
+			it "must be flagged as invalid" do
+				page.invalid.must_equal ""
+			end
+
+		end
+
+	end
+
+end
+

data/spec/lib/mediawiki-keiki/query_spec.rb

@@ -0,0 +1,135 @@
+require_relative '../../spec_helper'
+
+describe MediaWiki::Query do
+
+	describe "default attributes" do
+
+		it "must include httparty methods" do
+			MediaWiki::Query.must_include HTTParty
+		end
+
+		it "must have the base url set to the Wikipedia API endpoint" do
+			MediaWiki::Query.base_uri.must_equal 'https://en.wikipedia.org'
+		end
+
+		it "must have the User-Agent header" do
+			MediaWiki::Query.headers["User-Agent"].must_equal "crawler/1.0 (https://github.com/mkulumadzi/crawler)"
+		end
+
+	end
+
+	describe "default instance attributes" do
+
+		let(:wiki_query) { MediaWiki::Query.new('foo')}
+
+		it "must have a query" do
+			wiki_query.must_respond_to :query
+		end
+
+		it "must have the right query" do
+			wiki_query.query.must_equal 'foo'
+		end
+
+	end
+
+	describe "GET site" do
+
+		let(:wiki_query) { MediaWiki::Query.new('foo') }
+
+		before do
+			VCR.insert_cassette 'wiki_query', :record => :new_episodes
+		end
+
+		after do
+			VCR.eject_cassette
+		end
+
+		it "records the fixture" do
+			MediaWiki::Query.get('/w/api.php?continue=&format=json&action=query&titles=foo&prop=revisions&rvprop=content&redirects')
+		end
+
+		it "must have a query result method" do
+			wiki_query.must_respond_to :query_result
+		end
+
+		it "must parse the api response from JSON to Hash" do
+			wiki_query.query_result.must_be_instance_of Hash
+		end
+
+		it "must perform the request and get data" do
+			wiki_query.query_result["batchcomplete"].must_equal ""
+		end
+
+		describe "dynamic attributes" do
+
+			before do
+				wiki_query.query_result
+			end
+
+			it "must return the pages as a hash" do
+				wiki_query.pages.must_be_instance_of Hash
+			end
+
+			it "must use keys in pages that match the original search terms" do
+				wiki_query.pages.keys[0].must_equal 'foo'
+			end
+
+			it "must store the pages as Site classes" do
+				wiki_query.pages['foo'].must_be_instance_of MediaWiki::Page
+			end
+
+		end
+
+		describe "caching" do
+
+			# Use Webmock to disable the network connection after fetching the profile
+			before do
+				wiki_query.query_result
+				stub_request(:any, /en.wikipedia.org/).to_timeout
+			end
+
+			it "must cache the query result" do
+				wiki_query.query_result.must_be_instance_of Hash
+			end
+
+			it "must refresh the profile if forced" do
+				lambda { wiki_query.query_result(true) }.must_raise Timeout::Error
+			end
+
+		end
+
+		describe "multiple sites" do
+
+			let(:wiki_query) { MediaWiki::Query.new('foo|bar|camp') }
+
+			it "must return all of the sites" do
+				wiki_query.pages.length.must_equal 3
+			end
+
+			it "must tag the sites with the right query terms" do
+				wiki_query.pages['foo'].title.must_equal 'Foobar'
+				wiki_query.pages['bar'].title.must_equal 'Bar'
+				wiki_query.pages['camp'].title.must_equal 'Camp'
+			end
+
+		end
+
+		describe "give warning if WikiMedia API limit of 50 sites exceeded" do
+
+			search_string = ""
+
+			("1".."51").each { |x| search_string << x + "|"}
+
+			search_string = search_string.chomp("|")
+
+			it "must throw an error if search string has more than 50 sites" do
+				assert_raises ArgumentError do
+					MediaWiki::Query.new("#{search_string}".to_s)
+				end
+			end
+
+		end
+
+	end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+#Load the MediaWiki file
+require_relative '../lib/mediawiki-keiki'
+
+#dependencies
+require 'minitest/autorun'
+require 'webmock/minitest'
+require 'vcr'
+require 'pry'
+require 'minitest/reporters'
+
+#Minitest reporter 
+reporter_options = { color: true}
+Minitest::Reporters.use! [Minitest::Reporters::DefaultReporter.new(reporter_options)]
+ 
+#VCR config
+VCR.configure do |c|
+  c.cassette_library_dir = 'spec/fixtures/mediawiki_keiki_cassettes'
+  c.hook_into :webmock
+end

metadata

@@ -0,0 +1,201 @@
+--- !ruby/object:Gem::Specification
+name: mediawiki-keiki
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Evan Waters
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.13.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.13.3
+- !ruby/object:Gem::Dependency
+  name: wikicloth
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.5
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.20.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.20.4
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.3
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.7
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.24
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.24
+description: A client for the MediaWiki API
+email:
+- evan.waters@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- Rakefile
+- LICENSE
+- README.md
+- Gemfile
+- lib/mediawiki-keiki.rb
+- lib/mediawiki-keiki/page.rb
+- lib/mediawiki-keiki/query.rb
+- spec/spec_helper.rb
+- spec/lib/mediawiki-keiki/page_spec.rb
+- spec/lib/mediawiki-keiki/query_spec.rb
+homepage: https://github.com/mkulumadzi/mediawiki-keiki
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: MediaWiki Keiki
+test_files: []