marc2linkeddata 0.0.1

data/lib/marc2linkeddata/resource.rb

@@ -0,0 +1,131 @@
+
+module Marc2LinkedData
+
+  class Resource
+
+    attr_accessor :iri
+    # attr_reader :config
+
+    @@config = nil
+
+    def initialize(uri=nil)
+      @@config ||= Marc2LinkedData.configuration
+      if uri =~ /\A#{URI::regexp}\z/
+        uri = Addressable::URI.parse(uri.to_s) rescue nil
+      end
+      # Strip off any trailing '/'
+      if uri.to_s.end_with? '/'
+        uri = uri.to_s.gsub(/\/$/,'')
+        uri = Addressable::URI.parse(uri.to_s) rescue nil
+      end
+      raise 'invalid uri' unless uri.instance_of? Addressable::URI
+      @iri = uri
+    end
+
+    def id
+      @iri.basename
+    end
+
+    # This method is often overloaded in subclasses because
+    # RDF services use variations in the URL 'extension' patterns; e.g.
+    # see Loc#rdf and Viaf#rdf
+    def rdf
+      return @rdf unless @rdf.nil?
+      # TODO: try to retrieve the rdf from a local triple store
+      # TODO: if local triple store fails, try remote source(s)
+      # TODO: if retrieved from a remote source, save the rdf to a local triple store
+      @rdf = get_rdf(@iri.to_s)
+    end
+
+    def get_rdf(uri4rdf)
+      tries = 0
+      begin
+        tries += 1
+        @rdf = RDF::Graph.load(uri4rdf)
+      rescue
+        retry if tries <= 2
+        binding.pry if @@config.debug
+        nil
+      end
+    end
+
+    def rdf_uri
+      RDF::URI.new(@iri)
+    end
+
+    def rdf_valid?
+      iri_types.length > 0
+    end
+
+    def iri_types
+      q = SPARQL.parse("SELECT * WHERE { <#{@iri}> a ?o }")
+      rdf.query(q)
+    end
+
+    def rdf_find_object(id)
+      # TODO: convert this to an RDF.rb graph query?
+      return nil unless rdf_valid?
+      rdf.each_statement do |s|
+        if s.subject == @iri.to_s
+          return s.object if s.object.to_s =~ Regexp.new(id, Regexp::IGNORECASE)
+        end
+      end
+      nil
+    end
+
+    def rdf_find_subject(id)
+      # TODO: convert this to an RDF.rb graph query?
+      return nil unless rdf_valid?
+      rdf.each_statement do |s|
+        return s.subject if s.subject.to_s =~ Regexp.new(id, Regexp::IGNORECASE)
+      end
+      nil
+    end
+
+    def resolve_external_auth(url)
+      begin
+        res = Marc2LinkedData.http_head_request(url)
+        case res.code
+          when '200'
+            @@config.logger.debug "Mapped #{@iri}\t-> #{url}"
+            return url
+          when '301'
+            #301 Moved Permanently
+            url = res['location']
+            @@config.logger.debug "Mapped #{@iri}\t-> #{url}"
+            return url
+          when '302','303'
+            #302 Moved Temporarily
+            #303 See Other
+            # Use the current URL, most get requests will follow a 302 or 303
+            @@config.logger.debug "Mapped #{@iri}\t-> #{url}"
+            return url
+          when '404'
+            @@config.logger.warn "#{@iri}\t// #{url}"
+            return nil
+          else
+            # WTF
+            binding.pry if @@config.debug
+            @@config.logger.error "unknown http response code (#{res.code}) for #{@iri}"
+            return nil
+        end
+      rescue
+        nil
+      end
+    end
+
+    def same_as
+      same_as_url = 'http://sameas.org/rdf?uri=' + URI.encode(@iri.to_s)
+      RDF::Graph.load(same_as_url)
+    end
+
+    def same_as_array
+      q = SPARQL.parse("SELECT * WHERE { <#{@iri}> <http://www.w3.org/2002/07/owl#sameAs> ?o }")
+      same_as.query(q).collect {|s| s[:o] }
+    end
+
+  end
+
+end
+
+

data/lib/marc2linkeddata/sparql.rb

@@ -0,0 +1,55 @@
+require 'base64'
+
+module Marc2LinkedData
+
+  class Sparql
+
+    # https://github.com/ruby-rdf/sparql-client
+
+    # attr_reader :config
+    attr_reader :dbpedia
+    attr_reader :local_loc
+
+    def initialize
+      config = Marc2LinkedData.configuration
+      @dbpedia = SPARQL::Client.new('http://dbpedia.org/sparql')
+      # local LOC SPARQL client
+      auth_code = Base64.encode64("#{config.local_loc_user}:#{config.local_loc_pass}").chomp
+      headers = {
+          'Accept' => 'application/sparql-results+json',
+          'Authorization' => "Basic #{auth_code}",
+      }
+      @local_loc = SPARQL::Client.new(config.local_loc_url, {headers: headers} )
+    end
+
+    def local_loc_auth(auth_uri)
+      result = local_loc.query("SELECT * WHERE { <#{auth_uri}> ?p ?o }")
+      result.each_solution {|s| puts s.inspect }
+      binding.pry
+    end
+
+    # def sparql_dbpedia(query)
+    #   dbpedia.query(query)
+    #   # result = dbpedia.query('ASK WHERE { ?s ?p ?o }')
+    #   # puts result.inspect   #=> true or false
+    #   # result = dbpedia.query('SELECT * WHERE { ?s ?p ?o } LIMIT 10')
+    #   # result.each_solution {|s| puts s.inspect }
+    # end
+
+
+    # For reference, note that there is an allegrograph ruby gem, see
+    # https://github.com/emk/rdf-agraph
+
+    # For reference, note that there is a ruby gem for RDF on mongodb, see
+    # https://rubygems.org/gems/rdf-mongo
+
+
+
+
+
+
+  end
+
+end
+
+

data/lib/marc2linkeddata/viaf.rb

@@ -0,0 +1,48 @@
+require_relative 'resource'
+
+module Marc2LinkedData
+
+  class Viaf < Resource
+
+    PREFIX = 'http://viaf.org/viaf/'
+
+    # def id
+    #   return nil if iri.nil?
+    #   iri.path.gsub('viaf/','').gsub('/','')
+    # end
+
+    def rdf
+      return nil if iri.nil?
+      return @rdf unless @rdf.nil?
+      uri4rdf = iri.to_s + '/rdf.xml'
+      @rdf = get_rdf(uri4rdf)
+    end
+
+    def get_isni
+      return nil if iri.nil?
+      return nil unless rdf_valid?
+      return @isni_iri unless @isni_iri.nil?
+      # Try to get ISNI source for VIAF
+      # e.g. http://viaf.org/viaf/sourceID/ISNI%7C0000000109311081#skos:Concept
+      isni_iri = rdf_find_subject 'isni'
+      isni_src = URI.parse(isni_iri.to_s)
+      isni_iri = isni_src.path.sub('/viaf/sourceID/ISNI%7C','http://www.isni.org/isni/')
+      @isni_iri = resolve_external_auth(isni_iri)
+    end
+
+    def given_names
+      q = SPARQL.parse("SELECT * WHERE { <#{@iri}> <http://schema.org/givenName> ?o }")
+      names = rdf.query(q).collect {|s| s[:o].to_s}
+      names.to_set.to_a
+    end
+
+    def family_names
+      q = SPARQL.parse("SELECT * WHERE { <#{@iri}> <http://schema.org/familyName> ?o }")
+      names = rdf.query(q).collect {|s| s[:o].to_s}
+      names.to_set.to_a
+    end
+
+  end
+
+end
+

data/lib/marc2linkeddata.rb

@@ -0,0 +1,64 @@
+require 'dotenv'
+Dotenv.load
+
+require 'addressable/uri'
+require 'json'
+require 'linkeddata'
+require 'marc'
+require 'rdf/4store'
+require 'ruby-progressbar'
+
+require 'pry'
+require 'pry-doc'
+
+require_relative 'marc2linkeddata/configuration'
+
+require_relative 'marc2linkeddata/resource'
+require_relative 'marc2linkeddata/isni'
+require_relative 'marc2linkeddata/lib_auth'
+require_relative 'marc2linkeddata/loc'
+require_relative 'marc2linkeddata/viaf'
+
+require_relative 'marc2linkeddata/oclc_resource'
+require_relative 'marc2linkeddata/oclc_identity'
+require_relative 'marc2linkeddata/oclc_creative_work'
+require_relative 'marc2linkeddata/oclc_work'
+
+require_relative 'marc2linkeddata/parseMarcAuthority'
+#require_relative 'marc2linkeddata/parseMarcCatalog'
+require_relative 'marc2linkeddata/sparql'
+
+
+module Marc2LinkedData
+
+  # configuration at the module level, see
+  # http://brandonhilkert.com/blog/ruby-gem-configuration-patterns/
+
+  class << self
+    attr_writer :configuration
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.reset
+    @configuration = Configuration.new
+  end
+
+  def self.configure
+    yield(configuration)
+  end
+
+  def self.http_head_request(url)
+    uri = URI.parse(url)
+    Net::HTTP.start(uri.host, uri.port) {|http| req = Net::HTTP::Head.new(uri); http.request req }
+  end
+
+  def self.write_prefixes(file)
+    @configuration.prefixes.each_pair {|k,v| file.write "@prefix #{k}: <#{v}> .\n" }
+    file.write("\n\n")
+  end
+
+end
+

data/marc2linkeddata.gemspec

@@ -0,0 +1,40 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+Gem::Specification.new do |s|
+  s.name        = 'marc2linkeddata'
+  s.version     = '0.0.1'
+  s.licenses    = ['Apache-2.0']
+
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Darren Weber',]
+  s.email       = ['dlweber@stanford.edu']
+  s.summary     = 'Convert Marc21 records to linked data, for use in SUL/DLSS projects'
+  s.description = 'Utilities for translation of Marc21 records to linked open data.'
+  s.homepage    = 'https://github.com/darrenleeweber/marc2linkeddata'
+
+  s.required_rubygems_version = '>= 1.3.6'
+  s.required_ruby_version = '>= 2.1.0'
+
+  s.add_dependency 'addressable'
+  s.add_dependency 'linkeddata'
+  s.add_dependency 'marc'
+  s.add_dependency 'rdf-4store'
+  s.add_dependency 'ruby-progressbar'
+  s.add_dependency 'dotenv'
+
+  s.add_dependency 'hiredis'
+  s.add_dependency 'redis'
+
+  s.add_development_dependency 'pry'
+  s.add_development_dependency 'pry-doc'
+  s.add_development_dependency 'rspec'
+
+  s.files         = `git ls-files`.split($/)
+  dev_files = ['.gitignore','bin/setup.sh','bin/test.sh']
+  dev_files.each {|f| s.files.delete f }
+
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+end

data/spec/marc2linkeddata/configuration_spec.rb

@@ -0,0 +1,84 @@
+require "spec_helper"
+
+module Marc2LinkedData
+
+  describe Configuration do
+
+    describe '#debug' do
+      it 'default value is false' do
+        ENV['DEBUG'] = nil
+        config = Configuration.new
+        expect(config.debug).to be_falsey
+      end
+    end
+
+    describe '#debug=' do
+      it 'can set value' do
+        config = Configuration.new
+        config.debug = true
+        expect(config.debug).to be_truthy
+      end
+    end
+
+    describe '#redis4marc' do
+      it 'default value is false' do
+        config = Configuration.new
+        expect(config.redis4marc).to be_falsey
+      end
+    end
+
+    describe '#redis4marc=' do
+      it 'can set value' do
+        config = Configuration.new
+        config.redis4marc = true
+        expect(config.redis4marc).to be_truthy
+      end
+    end
+
+    describe '#redis_read' do
+      it 'default value is false' do
+        config = Configuration.new
+        expect(config.redis_read).to be_falsey
+      end
+    end
+
+    describe '#redis_read=' do
+      it 'can set value' do
+        config = Configuration.new
+        config.redis_read = true
+        expect(config.redis_read).to be_truthy
+      end
+    end
+
+    describe '#redis_write' do
+      it 'default value is false' do
+        config = Configuration.new
+        expect(config.redis_write).to be_falsey
+      end
+    end
+
+    describe '#redis_write=' do
+      it 'can set value' do
+        config = Configuration.new
+        config.redis_write = true
+        expect(config.redis_write).to be_truthy
+      end
+    end
+
+    describe '#prefixes' do
+      it 'default value is a hash' do
+        config = Configuration.new
+        expect(config.prefixes).to be_instance_of Hash
+      end
+    end
+
+    describe '#prefixes=' do
+      it 'can set value to hash' do
+        config = Configuration.new
+        config.prefixes = {}
+        expect(config.prefixes).to be_empty
+      end
+    end
+
+  end
+end

data/spec/marc2linkeddata/loc_spec.rb

@@ -0,0 +1,71 @@
+require "spec_helper"
+
+module Marc2LinkedData
+
+  describe Loc do
+
+    before :all do
+      # loc_urls = ['http://id.loc.gov/authorities/names/no99010609', 'http://id.loc.gov/authorities/names/no99010609/']
+      @loc_id = 'no99010609'
+      @loc_url = 'http://id.loc.gov/authorities/names/no99010609'
+      @loc = Loc.new @loc_url
+      @viaf_url = 'http://viaf.org/viaf/85312226'
+    end
+
+    before :each do
+    end
+
+    describe '#rdf' do
+      it 'should be an instance of RDF::Graph' do
+        expect(@loc.rdf.instance_of? RDF::Graph).to be_truthy
+      end
+    end
+
+    describe '#rdf_valid?' do
+      it 'should be true' do
+        expect(@loc.rdf_valid?).to be_truthy
+      end
+    end
+
+    describe '#same_as_array' do
+      it 'should be populated' do
+        expect(@loc.same_as_array.empty?).to be_falsey
+      end
+    end
+
+    describe '#get_viaf' do
+      it 'should equal the viaf url' do
+        expect(@loc.get_viaf).to eq(@viaf_url)
+      end
+    end
+
+    # TODO: add tests for different types of records, e.g.
+    # authorities:
+    #   person, organisation, conference, etc.
+
+
+    after :each do
+    end
+
+    after :all do
+      @loc_id = nil
+      @loc_url = nil
+      @loc = nil
+    end
+  end
+end
+
+
+# # valid data (Berners-Lee, Tim)
+# loc_iris = ['http://id.loc.gov/authorities/names/no99010609', 'http://id.loc.gov/authorities/names/no99010609/']
+# viaf_iri = 'http://viaf.org/viaf/85312226'
+# # valid data (Knuth, Donald Ervin)
+# # loc_iris = ['http://id.loc.gov/authorities/names/n79135509', 'http://id.loc.gov/authorities/names/n79135509/']
+# # viaf_iri = 'http://viaf.org/viaf/7466303'
+# loc_iris.each do |iri|
+#   id = Addressable::URI.parse(iri).basename
+#   loc = Marc2LinkedData::Loc.new iri
+# end
+# # invalid data
+# loc = Marc2LinkedData.Loc.new 'This is not an LOC IRI' rescue nil
+# raise "Loc.initialize failed to raise error." unless loc.nil?

data/spec/marc2linkeddata/resource_spec.rb

@@ -0,0 +1,53 @@
+require "spec_helper"
+
+module Marc2LinkedData
+
+  describe Resource do
+
+    before :all do
+      @auth_id = 'no99010609'
+      @auth_url = 'http://id.loc.gov/authorities/names/no99010609'
+      @auth = Resource.new @auth_url
+    end
+
+    before :each do
+    end
+
+    describe '#initialize' do
+      it 'should not raise error for a valid iri' do
+        # iri_valid = 'http://id.loc.gov/authorities/names/no99010609'
+        expect{Resource.new @auth_url}.not_to raise_error
+      end
+      it 'should raise error for an invalid iri' do
+        expect{Resource.new 'This is not a URL'}.to raise_error(RuntimeError)
+      end
+    end
+
+    describe '#id' do
+      it 'should equal the url basename' do
+        expect(@auth.id).to eq(@auth_id)
+      end
+    end
+
+    describe '#iri' do
+      it 'should equal the auth url' do
+        expect(@auth.iri.to_s).to eq(@auth_url)
+      end
+      it 'should be an instance of Addressable::URI' do
+        expect(@auth.iri.instance_of? Addressable::URI).to be_truthy
+      end
+    end
+
+    after :each do
+    end
+
+    after :all do
+      @auth_url = nil
+      @auth = nil
+    end
+
+  end
+
+end
+
+

data/spec/marc2linkeddata/viaf_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+module Marc2LinkedData
+
+  describe Viaf do
+
+    before :all do
+      # valid data (Knuth, Donald Ervin)
+      @viaf_id = '7466303'
+      @viaf_url = 'http://viaf.org/viaf/7466303'
+      @viaf = Viaf.new @viaf_url
+      @isni_url = 'http://www.isni.org/isni/000000012119421X'
+    end
+
+    before :each do
+    end
+
+    describe '#rdf' do
+      it 'should be an instance of RDF::Graph' do
+        expect(@viaf.rdf.instance_of? RDF::Graph).to be_truthy
+      end
+    end
+
+    describe '#rdf_valid?' do
+      it 'should be true' do
+        expect(@viaf.rdf_valid?).to be_truthy
+      end
+    end
+
+    describe '#same_as_array' do
+      it 'should be populated' do
+        expect(@viaf.same_as_array.empty?).to be_falsey
+      end
+    end
+
+    describe '#get_isni' do
+      it 'should equal the isni url' do
+        expect(@viaf.get_isni).to eq(@isni_url)
+      end
+    end
+
+    after :each do
+    end
+
+    after :all do
+      @viaf_id = nil
+      @viaf_url = nil
+      @isni_url = nil
+      @viaf = nil
+    end
+  end
+end
+

data/spec/marc2linkeddata_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe Marc2LinkedData do
+
+  describe "#configure" do
+    before :each do
+      Marc2LinkedData.configure do |config|
+        config.debug = true
+      end
+    end
+    it "returns a hash of options" do
+      config = Marc2LinkedData.configuration
+      expect(config).to be_instance_of Marc2LinkedData::Configuration
+      expect(config.debug).to be_truthy
+    end
+    after :each do
+      Marc2LinkedData.reset
+    end
+  end
+
+  describe ".reset" do
+    before :each do
+      Marc2LinkedData.configure do |config|
+        config.debug = true
+      end
+    end
+    it "resets the configuration" do
+      Marc2LinkedData.reset
+      config = Marc2LinkedData.configuration
+      expect(config).to be_instance_of Marc2LinkedData::Configuration
+      expect(config.debug).to be_falsey
+    end
+    after :each do
+      Marc2LinkedData.reset
+    end
+  end
+
+end
+