wikimedia-commoner 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a227a091c7462e7ee5cbc5a4f5006fed5d7178cc
+  data.tar.gz: e64c81d9b390d41cb4781828b2752462ebe3e4fd
+SHA512:
+  metadata.gz: 144dd7ff7a215bcb8ffb4d65e30bc8807d6ad3c6b1dab16fdaa72d4103ed7e074a693ff68d8979f83b8ab610b51c3f1704becb477f1415acb1d9ace351d55637
+  data.tar.gz: 9a2fbdee2ec60bdded019b15dfba94b7b17950d7099e8ba76e121af452319245b0a10df2f6cd029370a83b11faa67f676f59d96ebd36fbe3edcdd5ee458fad4d

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in wikimedia-commoner.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2016 Jez Nicholson
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+# Wikimedia::Commoner
+
+A Wikimedia Commons page holds more details than the api provides, e.g. the name of the copyright holder. This can be scraped from the html as a standard format is encouraged with mediawiki templates. It is not foolproof, but will work for most of the images.
+
+This gem is used by OpenPlaques.org to get hold of image links and appropriate copyright holder details so that an image can be legally displayed on a web page.
+
+The idea is that users can copy and paste any Wikimedia filename to your system and you can get the details. It looks for anything with 'File:' in the url, so can cope with different formats coming from preview urls, etc.:
+* https://commons.wikimedia.org/wiki/File:Ciconia_ciconia_-_01.jpg
+* https://commons.wikimedia.org/wiki/Main_Page#/media/File:Ciconia_ciconia_-_01.jpg
+* File:Ciconia_ciconia_-_01.jpg
+* https://commons.wikimedia.org/wiki/File:Plaque_in_Fortune_Street.jpg#.7B.7Bint:filedesc.7D.7D
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'wikimedia-commoner'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install wikimedia-commoner
+
+## Usage
+
+require 'wikimedia/commoner'
+
+wikimedia = Wikimedia::Commoner.details(wikimedia_url)
+wikimedia[:categories]
+wikimedia[:url]
+wikimedia[:page_url]
+wikimedia[:description]
+wikimedia[:author]
+wikimedia[:author_url]
+wikimedia[:licence]
+wikimedia[:licence_url])
+
+Wikimedia::Commoner.search 'term'
+
+Wikimedia::Commoner.images 'term'
+
+Wikimedia::Commoner.categorised_images 'category'
+
+## Contributing
+
+1. Fork it ( https://github.com/jnicho02/wikimedia-commoner/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+With thanks to Ross Cooperman for the inspiration with his 2013 commoner gem. I did try a pull request.

data/Rakefile

@@ -0,0 +1,19 @@
+#!/usr/bin/env rake
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+# Run with 'rake spec'
+RSpec::Core::RakeTask.new(:spec) do |task|
+#  task.rspec_opts = ['--color', '--format', 'nested']
+end
+
+# http://erniemiller.org/2014/02/05/7-lines-every-gems-rakefile-should-have/
+task :console do
+  require 'irb'
+  require 'irb/completion'
+  require 'wikimedia/commoner'
+  ARGV.clear
+  IRB.start
+end
+
+task :default => :test

data/lib/wikimedia/commoner.rb

@@ -0,0 +1,154 @@
+require 'wikimedia/commoner/version'
+require 'httparty'
+require 'sanitize'
+
+module Wikimedia
+  class Commoner
+    def self.search(query)
+      new.search(query)
+    end
+
+    def self.images(term)
+      new.images(term)
+    end
+
+    def self.categorised_images(term)
+      new.categorised_images(term)
+    end
+
+    def self.details(title)
+      title = URI.unescape title
+      new.details(title)
+    end
+
+    def initialize(base_uri = nil)
+      @uri = (base_uri) ? base_uri : default_uri
+    end
+
+    def search(query)
+      json_get(search_uri(query))[1]
+    end
+
+    def images(term)
+      # get a list of titles for the given term
+      response = json_get(query_uri(term))
+      images   = response['query']['pages'].map { |page_id, page| page['images'] }
+      if images!=[nil]
+        titles   = images.flatten.map { |image| image['title'] }
+        titles.map do |title|
+          details(title)
+        end
+      end
+    end
+
+    def categorised_images(category)
+      # get a list of titles for the given term
+      response = json_get(category_uri(category))
+      images = response['query']['categorymembers']
+      if images!=[nil]
+        titles = images.flatten.map { |image| image['title'] }
+        titles.map do |title|
+          if title.start_with?('Category:')
+            categorised_images(title)
+          else
+            details(title)
+          end
+        end
+      end
+    end
+
+    def details(title)
+      return nil if /File:.*/.match(title) == nil
+      title = /File:[^#]*/.match(title)[0]
+      response = json_get(info_uri(title))
+      return {} if response == nil
+      pages = response['query']['pages'].map { |page_id, page| page }
+      return { description: 'missing' } if pages.first['missing']!=nil
+      categories = pages.first['categories'].map { |category| category['title'] }.flatten
+      categories = categories.map { |category| category.gsub(/^Category:/, '') }
+      descriptionurl = pages.first['imageinfo'].first['descriptionurl']
+      licence = pages.first['imageinfo'].first['extmetadata']['LicenseShortName']['value']
+      licence_url = pages.first['imageinfo'].first['extmetadata']['LicenseUrl']['value'] if pages.first['imageinfo'].first['extmetadata']['LicenseUrl']
+      if categories.include? 'CC-PD-Mark'
+        licence = 'CC-PD-Mark'
+        licence_url = 'http://creativecommons.org/publicdomain/mark/1.0'
+      end
+      licence_url = 'https://en.wikipedia.org/wiki/Public_domain' if licence == 'Public domain' && licence_url == nil
+      party = HTTParty.get(descriptionurl, :verify => false)
+      doc = Nokogiri::HTML(party.to_s)
+      an = doc.xpath('//span[@id="creator"]')
+      author_name = an[0].content if !an.empty?
+      if an.empty?
+        an = doc.xpath('//tr[td/@id="fileinfotpl_aut"]/td')
+        author_name = an[1].content if !an.empty? && an.size > 0
+      end
+      author_name = Sanitize.clean(author_name)
+      author_url = ""
+      au = doc.xpath('//span[@id="creator"]/*/a/@href')
+      au = doc.xpath('//tr[td/@id="fileinfotpl_aut"]/td/a/@href') if au.empty?
+      author_url = au[0].content if !au.empty? && au.size > 0
+      author_url = "http://commons.wikimedia.org" + author_url if author_url.start_with?('/wiki/User:')
+      description = ""
+      description_element = doc.xpath('//td[@class="description"]')
+      description = Sanitize.clean(description_element[0].content)[0,255].strip! if description_element.size > 0
+
+      geohack = doc.xpath("//a[contains(@href, 'tools.wmflabs.org/geohack')]/@href")
+      longitude = geohack[0] ? /params=.*_[NS]_(-*\d*.\d*)/.match(geohack[0].value)[1] : nil
+      latitude = geohack[0] ? /params=(\d*.\d*)/.match(geohack[0].value)[1] : nil
+
+      openplaques_url = doc.xpath("//a[contains(@href, 'openplaques.org/plaques')]/@href")
+      openplaques_id = openplaques_url[0] ? /plaques\/(\d*)/.match(openplaques_url[0].value)[1] : nil
+
+      page_url = "https://commons.wikimedia.org/wiki/"+title
+      {
+        categories:  categories,
+        url:         pages.first['imageinfo'].first['url'],
+        page_url:    page_url,
+        description: description,
+        author:      author_name,
+        author_url:  author_url,
+        licence:     licence,
+        licence_url: licence_url,
+        longitude:   longitude,
+        latitude:    latitude,
+        openplaques_id: openplaques_id
+      }
+    end
+
+    private
+
+      def json_get(uri)
+        response = HTTParty.get(uri, :verify => false)
+        if response.code == 200
+          JSON.parse(response.body)
+        else
+          raise Exception.new("bad status code from Wikimedia server (#{response.code}):\n#{response.body}")
+        end
+      end
+
+      def uri_for(params)
+        "#{@uri}?#{URI.encode_www_form(params)}"
+      end
+
+      def search_uri(query)
+        uri_for action: 'opensearch', search: query, limit: 100
+      end
+
+      def query_uri(term)
+        uri_for action: 'query', titles: term, prop: 'images', format: 'json'
+      end
+
+      def info_uri(image)
+        uri_for action: 'query', titles: image, prop: 'imageinfo|categories', iiprop: 'url|extmetadata', format: 'json'
+      end
+
+      def category_uri(term)
+        uri_for action: 'query', list: 'categorymembers', format: 'json', cmtitle: term
+      end
+
+      def default_uri
+        @default_uri ||= "https://commons.wikimedia.org/w/api.php"
+      end
+
+    end
+end

data/lib/wikimedia/commoner/version.rb

@@ -0,0 +1,5 @@
+module Wikimedia
+  class Commoner
+    VERSION = "1.0.0"
+  end
+end

data/spec/author_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+
+describe Wikimedia::Commoner do
+
+  describe '#author' do
+    context 'of a Delhi portrait of a man' do
+      it 'is Jorge Royan' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'https://commons.wikimedia.org/wiki/File:India_-_Delhi_portrait_of_a_man_-_4780.jpg'
+          expect(image[:author]).to eq 'Jorge Royan'
+        end
+      end
+    end
+    context 'of Nahal Zaror, south 11' do
+      it 'is Daniel Baránek' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'File:Nahal_Zaror,_south_11.jpg'
+          expect(image[:author]).to eq 'Daniel Baránek'
+        end
+      end
+      it 'is linked to Daniel Baránek wikimedia user page' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'File:Nahal_Zaror,_south_11.jpg'
+          expect(image[:author_url]).to eq 'http://commons.wikimedia.org/wiki/User:Daniel_Bar%C3%A1nek'
+        end
+      end
+    end
+    context 'of The mohave desert near the fossil beds' do
+      it 'is unknown' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'File:PSM_V86_D252_The_mohave_desert_near_the_fossil_beds.jpg'
+          expect(image[:author]).to eq 'Unknown'
+        end
+      end
+    end
+    context 'of a Spanish Civil War mass grave' do
+      it 'is Mario Modesto Mata' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'File:Spanish_Civil_War_-_Mass_grave_-_Estépar,_Burgos.jpg'
+          expect(image[:author]).to eq 'Mario Modesto Mata'
+        end
+      end
+    end
+    context 'of August Wilhelm von Hofmann' do
+      it 'is unknown' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'File:Hoffman_August_Wilhelm_von.jpg'
+          expect(image[:author]).to eq 'Unknown'
+        end
+      end
+      it 'is not linked to an author url' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'File:Hoffman_August_Wilhelm_von.jpg'
+          expect(image[:author_url]).to eq ''
+        end
+      end
+    end
+    context 'of Catherine of Aragon portrait' do
+      it 'is Lucas Hornebolte' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'File:Catherine_aragon.jpg'
+          expect(image[:author]).to eq 'Lucas Hornebolte'
+        end
+      end
+    end
+    context 'of Stepan Blois portrait' do
+      it 'is Matthew Paris' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'https://commons.wikimedia.org/wiki/File:Stepan_Blois.jpg'
+          expect(image[:author]).to eq 'Matthew Paris'
+        end
+      end
+      it 'is linked to Matthew Paris Wikipedia page' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'https://commons.wikimedia.org/wiki/File:Stepan_Blois.jpg'
+          expect(image[:author_url]).to eq '//en.wikipedia.org/wiki/Matthew_Paris'
+        end
+      end
+    end
+  end
+
+end

data/spec/commoner_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe Wikimedia::Commoner do
+
+  describe '#searching' do
+    context 'on an unknown term' do
+      it 'finds nothing' do
+        VCR.use_cassette ('searching/' + self.class.description).gsub(" ","-") do
+          titles = Wikimedia::Commoner.search 'badger'
+          expect(titles).to eq([])
+        end
+      end
+    end
+    context 'on a known term' do
+      it 'finds some titles' do
+        VCR.use_cassette ('searching/' + self.class.description).gsub(" ","-") do
+          titles = Wikimedia::Commoner.search 'Meles meles'
+          expect(titles.size).to be > 0
+        end
+      end
+    end
+  end
+
+  describe '#images' do
+    context 'on a known term' do
+      it 'finds at least one image' do
+        VCR.use_cassette ('images/' + self.class.description).gsub(" ","-") do
+        	images = Wikimedia::Commoner.images 'Meles meles'
+          first = images[0]
+          expect(first[:url].start_with?("https:")).to be(true)
+        end
+      end
+    end
+    context 'on an unknown term' do
+      it 'finds nothing' do
+        VCR.use_cassette ('images/' + self.class.description).gsub(" ","-") do
+          images = Wikimedia::Commoner.images 'plaques'
+          expect(images).to eq nil
+        end
+      end
+    end
+  end
+
+  describe '#categorised_images' do
+    context 'a base category' do
+      it 'lists images categorised as this and within subcategories' do
+        VCR.use_cassette ('categorised_images/' + self.class.description).gsub(" ","-") do
+          images = Wikimedia::Commoner.categorised_images 'Category:Stolperstein-Plagiate'
+        end
+      end
+    end
+  end
+end

data/spec/details_spec.rb

@@ -0,0 +1,87 @@
+require 'spec_helper'
+
+describe Wikimedia::Commoner do
+
+  describe '#details' do
+    context 'of a non-file page name' do
+      it 'gives details of a title' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          images = Wikimedia::Commoner.details 'Meles meles'
+        end
+      end
+    end
+    context 'of an unknown page name' do
+      it 'gives details of a title' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'File:Does_not_exist.jpg'
+          expect(image[:description]).to eq 'missing'
+        end
+      end
+    end
+    context 'of a Commons file page' do
+      it 'gives details of a title' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'File:Badger 25-07-09.jpg'
+        end
+      end
+      it 'gives has a page url' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'File:Badger 25-07-09.jpg'
+          expect(image[:page_url]).to eq 'https://commons.wikimedia.org/wiki/File:Badger 25-07-09.jpg'
+        end
+      end
+    end
+    context 'of a full Commons file page url' do
+      it 'gives details of a title' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'https://commons.wikimedia.org/wiki/File:Badger 25-07-09.jpg'
+        end
+      end
+    end
+    context 'of a Wikipedia image preview url' do
+      it 'gives details of a title' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'https://en.wikipedia.org/wiki/Main_Page#mediaviewer/File:Suillus_pungens_123004.jpg'
+        end
+      end
+    end
+    context 'of a Wikipedia mobile image preview url' do
+      it 'gives details of a title' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'https://en.m.wikipedia.org/wiki/Main_Page#mediaviewer/File:Suillus_pungens_123004.jpg'
+        end
+      end
+    end
+    context 'of url with a strange character in' do
+      it 'gives details of a title' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'https://commons.wikimedia.org/wiki/File:Jacobäerschild_in_Bautzen.JPG'
+        end
+      end
+    end
+    context 'of url for Jacob with unicode in' do
+      it 'gives details of a title' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'https://commons.wikimedia.org/wiki/File:Jacob%C3%A4erschild_in_Bautzen.JPG'
+        end
+      end
+    end
+    context 'of a deleted photo' do
+      it 'gives a description of "missing"' do
+        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'https://commons.wikimedia.org/wiki/File:Vancouver_-_Gastown_Steam_Clock_plaque.jpg'
+          expect(image[:description]).to eq 'missing'
+        end
+      end
+    end
+    context 'of a photo with extra bits on the end of the url' do
+      it 'gives a description of "missing"' do
+#        VCR.use_cassette ('details/' + self.class.description).gsub(" ","-") do
+          image = Wikimedia::Commoner.details 'https://commons.wikimedia.org/wiki/File:Plaque_in_Fortune_Street.jpg#.7B.7Bint:filedesc.7D.7D'
+          expect(image[:page_url]).to eq 'https://commons.wikimedia.org/wiki/File:Plaque_in_Fortune_Street.jpg'
+#        end
+      end
+    end
+  end
+
+end