doctor_scrape 0.0.2

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/ruby

data/.rbenv-version

@@ -0,0 +1 @@
+1.9.3-p125-perf

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in doctor_scrape.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,10 @@
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/doctor_scrape/scraper/(.+)\.rb$})  { |m| "spec/scrapers/#{m[1]}_spec.rb" }
+  watch(%r{^spec/shared/})      { "spec" }
+  watch("spec/spec_helper.rb")  { "spec" }
+  watch(%r{^lib/.+\.rb$})       { "spec" }
+end
+

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Gudleik Rasch
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,78 @@
+# DoctorScrape
+
+Collection of libraries to scrape contents from norwegian doctoral dissertations.
+Used by http://avhandlinger.no
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'doctor_scrape'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install doctor_scrape
+
+## Usage
+
+Ta-da
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Resources
+
+### Supported
+
+* Duo: http://www.duo.uio.no/sok/search.html?documentTypes=Doktoravhandling&yearFrom=1917&yearTo=2011
+* Brage/NHH: http://brage.bibsys.no/nhh/browse?type=type&order=ASC&rpp=250&value=Doctoral+thesis
+* DIVA/NTNU: http://ntnu.diva-portal.org/smash/searchlist.jsf?searchId=1
+* MUNIN/UiT: http://www.ub.uit.no/munin/
+* Bora/HiB: https://bora.hib.no/
+
+### Unsupported
+
+DE STORE
+
+* Bora/UiB: https://bora.uib.no/browse?type=documenttype&order=ASC&rpp=650&value=Doctoral+thesis
+* TEORA: http://teora.hit.no/dspace/ (Merk at avhandlinger ligger under ulike enheter, ikke bare «Doktorgradsavhandlinger - dr. ingeniør».) 
+
+DE MELLOMSTORE
+
+* Brage/BI: http://brage.bibsys.no/bi/browse?type=type&order=ASC&rpp=50&value=Doctoral+thesis
+* Brage/NIH: http://brage.bibsys.no/nih/browse?type=type&order=ASC&rpp=500&value=Doctoral+thesis
+* Brage/UiA: http://brage.bibsys.no/hia/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/UiN: http://brage.bibsys.no/hibo/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/UiS: http://brage.bibsys.no/uis/browse?type=type&order=ASC&rpp=25&value=Doctoral+thesis
+
+DE SMÅ
+
+* ODA/HiO: https://oda.hio.no/jspui/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/AHO: http://brage.bibsys.no/aho/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/DHS: http://brage.bibsys.no/diakon/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/HSF: http://brage.bibsys.no/hsf/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/HiNT: http://brage.bibsys.no/hint/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/HiNe: http://brage.bibsys.no/hinesna/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/HiL: http://brage.bibsys.no/hil/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/HiST: http://brage.bibsys.no/hist/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/HiL: http://brage.bibsys.no/hil/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/MHS: http://brage.bibsys.no/misjon/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/PHS: http://brage.bibsys.no/politihs/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/NP: http://brage.bibsys.no/npolar/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/HiH: http://brage.bibsys.no/hhe/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/IMR: http://brage.bibsys.no/imr/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/HiØ: http://brage.bibsys.no/hiof/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/NMH: http://brage.bibsys.no/nmh/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/SA: http://brage.bibsys.no/samall/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/UMB: http://brage.bibsys.no/umb/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/SSB: http://brage.bibsys.no/ssb/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis
+* Brage/KRUS: http://brage.bibsys.no/krus/browse?type=type&order=ASC&rpp=20&value=Doctoral+thesis

data/Rakefile

@@ -0,0 +1,14 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+desc "run specs"
+RSpec::Core::RakeTask.new
+
+task :default => :spec
+
+desc "Run test suite and generate coverage report"
+task :coverage do
+  ENV["SIMPLECOV"] = "1"
+  Rake::Task['spec'].invoke
+end

data/doctor_scrape.gemspec

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/doctor_scrape/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Gudleik Rasch"]
+  gem.email         = ["gudleik@gmail.com"]
+  gem.description   = "Library for scraping norwegian doctoral dissertations"
+  gem.summary       = "Library for scraping norwegian doctoral dissertations"
+  gem.homepage      = "https://github.com/Skalar/doctor_scrape"
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "doctor_scrape"
+  gem.require_paths = ["lib"]
+  gem.version       = DoctorScrape::VERSION
+
+  gem.add_dependency "mechanize", ["~> 2.3"]
+  # gem.add_dependency "text", ["~> 1.0.3"]
+
+  gem.add_development_dependency "rspec", ["~> 2.8.0"]
+  gem.add_development_dependency "vcr", ["~> 2.0.0.rc1"]
+  gem.add_development_dependency "webmock", ["< 1.8"]
+  gem.add_development_dependency "guard", ["~> 1.0.0" ]
+  gem.add_development_dependency "guard-rspec", ["~> 0.6.0"]
+  gem.add_development_dependency "ruby_gntp", ["~> 0.3.4"]
+  gem.add_development_dependency "rb-fsevent", ["~> 0.9.0"]
+  gem.add_development_dependency "pry", ["~> 0.9.8.2"]
+  gem.add_development_dependency "pry-doc", ["~> 0.4.0"]
+  gem.add_development_dependency "pry-editline", ["~> 1.1.1"]
+  gem.add_development_dependency "hirb", ["~> 0.6.0"]
+  gem.add_development_dependency "awesome_print", ["~> 1.0.2"]
+  gem.add_development_dependency "simplecov", ["~> 0.6.0"]
+  # gem.add_development_dependency "psych", ["~> 1.2.2"]
+end

data/lib/doctor_scrape/data.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+module DoctorScrape
+  class Data < OpenStruct
+    def fields
+      @table.keys
+    end
+
+    def to_hash
+      @table
+    end
+  end
+end

data/lib/doctor_scrape/redirect_follower.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require 'net/https'
+module DoctorScrape
+  class TooManyRedirects < StandardError; end
+
+  class RedirectFollower
+    attr_accessor :url
+
+    def initialize(url)
+      @url = url
+    end
+
+    def resolve(limit=5)
+      raise TooManyRedirects if limit == 0
+
+      response = Net::HTTP.get_response URI.parse(@url)
+
+      if response.is_a? Net::HTTPRedirection
+        @url = response['location']
+        resolve limit - 1
+      end
+
+      @url
+    rescue Net::HTTPBadResponse => error
+      # This can safely be ignored
+    end
+
+  end
+end

data/lib/doctor_scrape/scraper/base.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+module DoctorScrape::Scraper
+
+  class Base
+    attr_reader :data, :errors
+
+    def initialize(url)
+      @url   = url
+      @data  = DoctorScrape::Data.new url: @url
+    end
+
+    def scrape
+      @errors = []
+      fetch && parse
+    rescue => error
+      @errors << error
+    ensure
+      return !errors?
+    end
+
+    def fetch
+      @body ||= open(url)
+    end
+
+    def doc
+      @doc ||= Nokogiri::HTML @body
+    end
+
+    def url
+      @url
+    end
+
+    def errors?
+      errors.any?
+    end
+
+    protected
+
+    def link(selector)
+      href = doc.at(selector)['href']
+      href ? URI.parse(@url).merge(href).to_s : nil
+    end
+
+    def pdf_from_link
+      link ".standard:nth-child(1) a"
+    end
+
+  end
+
+end

data/lib/doctor_scrape/scraper/bora.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+module DoctorScrape::Scraper
+
+  class Bora < Base
+
+    def parse
+
+      doc.search("table.itemDisplayTable tr").each do |el|
+        key, value = el.children[0..1].map &:text
+        next unless key =~ /^dc\./
+
+        attribute = key.match(/\.(?<attr>[a-z]+)$/)[:attr]
+        @data.send("#{attribute}=", value)
+      end
+
+      @data.permalink = @data.uri
+      @data.pdf       = pdf_from_link
+    end
+
+    def url
+      "#{@url}?mode=full"
+    end
+  end
+
+end

data/lib/doctor_scrape/scraper/diva.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+module DoctorScrape::Scraper
+
+  class Diva < Meta
+
+    def parse
+      @data.title     = text "DC.Title"
+      @data.author    = text "DC.Creator"
+      @data.issued    = text "DC.Date"
+      @data.permalink = text "DC.Identifier.url"
+      @data.pdf       = text "citation_pdf_url"
+    end
+
+  end
+
+end

data/lib/doctor_scrape/scraper/duo.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+module DoctorScrape::Scraper
+
+  class Duo < Base
+
+    def parse
+      doc.search("#main_container table:nth-child(2) tr").each do |el|
+        label, value = el.children.search("td")[0..1].map { |a| a.text.strip.gsub(/\r\n/, '') }
+
+        case label
+        when /Tittel/     then @data.title  = value
+        when /Forfatter/  then @data.author = value
+        when /Publisert/  then @data.issued = value
+        when /Permanent/  then @data.permalink = value
+        end
+
+      end
+
+      @data.pdf       = link "#main_container table:nth-child(2) + p a"
+      @data.abstract  = text "#main_container p:last"
+    end
+
+    protected
+
+    def text(selector)
+      el = doc.at(selector)
+      el ? el.text.strip : nil
+    end
+
+  end
+
+end

data/lib/doctor_scrape/scraper/meta.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+module DoctorScrape::Scraper
+
+  # This class uses the meta tags from the HTML document to get the content
+  class Meta < Base
+
+    def parse
+      @data.title     = text "DC.title"
+      @data.author    = text "DC.creator"
+      @data.issued    = text "DCTERMS.issued"
+      @data.permalink = doc.xpath("//meta[@scheme='DCTERMS.URI']/@content").text
+      @data.pdf       = pdf_from_meta || pdf_from_link
+      @data.abstract  = abstract
+    end
+
+    protected
+
+    def abstract
+      meta("DCTERMS.abstract").map(&:text).sort { |a,b| b.size <=> a.size }.first
+    end
+
+    def text(name)
+      if value = meta(name).text
+        CGI.unescapeHTML(value).gsub(/\r\n/, ' ')
+      end
+    end
+
+    def meta(name)
+      doc.xpath("//meta[@name='#{name}']/@content")
+    end
+
+    def pdf_from_meta
+      value = doc.xpath("//meta[contains(@content, 'pdf')]/@content").map(&:text).select { |x| x =~ /http/ }.first 
+      # sometimes the value contains Fulltext, doh
+      value.gsub(/Fulltext /, '') if value
+    end
+
+  end
+end

data/lib/doctor_scrape/scraper/unknown.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+module DoctorScrape::Scraper
+
+  class Unknown < Base
+
+    def errors
+      [ "Don't know how to scrape this page" ]
+    end
+
+    def scrape
+      false
+    end
+
+    def fetch
+      false
+    end
+
+  end
+
+end

data/lib/doctor_scrape/search.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+require 'rss'
+
+module DoctorScrape
+
+  class Search
+
+    class << self
+      def nora(options={})
+        params = {
+          "PAGESIZE"       => options[:limit] || 50,
+          "FROM"           => options[:from]  || 2007,
+          "TO"             => options[:to]    || Time.now.year,
+          "SEARCHMODE"     => "TOPIC",
+          "DOCUMENTTYPES"  => "Doctoral+thesis",
+          "RESULTMODE"     => "rss",
+        }
+
+        url = "http://www.ub.uio.no/nora/result.html?" << params.map { |key,val| "#{key}=#{val}" }.join("&")
+
+        open url do |rss|
+          feed = RSS::Parser.parse rss
+
+          return [] if feed.nil?
+          items = feed.items.map &:link
+
+          options[:resolve] ? resolve_scrapers(items) : items
+        end
+      end
+
+      def resolve_urls(urls)
+        urls.map { |url| RedirectFollower.new(url).resolve }
+      end
+
+      def resolve_scrapers(urls)
+        resolve_urls(urls).map { |url| Scraper.for url }
+      end
+
+    end
+  end
+
+end

data/lib/doctor_scrape/version.rb

@@ -0,0 +1,3 @@
+module DoctorScrape
+  VERSION = "0.0.2"
+end

data/lib/doctor_scrape.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require 'doctor_scrape/version'
+require 'mechanize'
+require 'nokogiri'
+require 'ostruct'
+require 'open-uri'
+
+module DoctorScrape
+  autoload :Data,             'doctor_scrape/data'
+  autoload :Search,           'doctor_scrape/search'
+  autoload :RedirectFollower, 'doctor_scrape/redirect_follower'
+
+  module Scraper
+    autoload :Base,    'doctor_scrape/scraper/base'
+    autoload :Bora,    'doctor_scrape/scraper/bora'
+    autoload :Duo,     'doctor_scrape/scraper/duo'
+    autoload :Diva,    'doctor_scrape/scraper/diva'
+    autoload :Meta,    'doctor_scrape/scraper/meta'
+    autoload :Unknown, 'doctor_scrape/scraper/unknown'
+
+    # TODO: move this logic into the scraper classes
+    def self.for(url)
+      case url
+      when %r{^http://(www.)?duo.uio.no}                             then Duo.new  url
+      when %r{^https?://bora.hib.no}                          then Bora.new url
+      when %r{^http://.+bibsys.no},
+           %r{idtjeneste.nb.no/URN:NBN:no-bibsys_brage}       then Meta.new url
+      when %r{^http://ntnu.diva-portal.org/smash/record.jsf},
+           %r{urn=urn:nbn:no:ntnu:diva}                       then Diva.new url
+
+      else
+        Unknown.new url
+      end
+    end
+  end
+
+end