marc2linkeddata 0.0.1

data/bin/readMarcAuthority

@@ -0,0 +1,113 @@
+#!/usr/bin/env ruby
+
+require 'marc2linkeddata'
+
+CONFIG = Marc2LinkedData.configuration
+
+def marc_auth_count(marc_file)
+  auth_records = 0
+  until marc_file.eof?
+    begin
+      leader = Marc2LinkedData::ParseMarcAuthority::parse_leader(marc_file)
+      marc_file.seek(leader[:length], IO::SEEK_CUR)
+      auth_records += 1 if leader[:type] == 'z'
+    rescue => e
+      puts
+      puts 'ERROR'
+      puts e.message
+      puts e.backtrace
+      puts
+      binding.pry if CONFIG.debug
+    end
+  end
+  marc_file.seek(0, IO::SEEK_SET)
+  auth_records
+end
+
+def marc2ld(marc_filename)
+  ld_filename = marc_filename.gsub('.mrc','.ttl')
+  puts "Translating: #{marc_filename} to #{ld_filename}"
+  ld_file = File.open(ld_filename,'w')
+  # Marc2LinkedData.write_prefixes(ld_file)
+  marc_file = File.open(marc_filename,'r')
+  auth_count = 0
+  auth_records = marc_auth_count(marc_file)
+  progress = ProgressBar.create(:total => auth_records, :format => '%a %f |%b>>%i| %P%% %t')
+  until marc_file.eof?
+    begin
+      leader = Marc2LinkedData::ParseMarcAuthority::parse_leader(marc_file)
+      raw = marc_file.read(leader[:length])
+      if leader[:type] == 'z'
+        progress.increment
+        record = MARC::Reader.decode(raw)
+        # ParseMarcAuthority is a lazy parser, so
+        # init only assigns record to an instance var.
+        auth = Marc2LinkedData::ParseMarcAuthority.new(record)
+        auth_count += 1
+        # auth_id = "auth:#{auth.get_id}"
+        # triples = nil
+        # # TODO: enable additional persistence options
+        # # Use data already in redis (if enabled)
+        # triples = CONFIG.redis.get(auth_id) if CONFIG.redis_read
+        # if triples.nil?
+        #   triples = auth.to_ttl  # generate new triples
+        #   # Update redis (if enabled) for triples not read from redis
+        #   CONFIG.redis.set(auth_id, triples) if CONFIG.redis_write
+        # end
+
+        triples = auth.to_ttl.lines
+        binding.pry if (CONFIG.debug && triples.empty?)
+        triples.delete_if {|l| l.chomp.empty? }
+        triples.delete_if {|l| l.start_with?('@prefix') } if auth_count > 1
+        ld_file.write(triples.join)
+        ld_file.flush
+      end
+    rescue => e
+      puts
+      puts 'ERROR'
+      puts e.message
+      puts e.backtrace
+      puts record.to_s
+      puts
+      binding.pry if CONFIG.debug
+    end
+  end
+  marc_file.close
+  ld_file.flush
+  ld_file.close
+end
+
+marc_files = []
+ARGV.each do |filename|
+  path = Pathname(filename)
+  marc_files.push(path) if path.exist?
+end
+if marc_files.empty?
+  puts <<HELP
+#{__FILE__} marc_authority_file1.mrc [ marc_authority_file2.mrc .. marc_authority_fileN.mrc ]
+
+Output is RDF triples in a turtle file (.ttl) for every input .mrc file.
+Optional persistence services can be controlled by environment variables.
+
+Redis Persistence - based on https://github.com/redis/redis-rb
+- essential options:
+  export REDIS4MARC=true # enable redis persistence (default = false)
+- supplementary options:
+  Set the REDIS_URL for a custom redis configuration.
+  export REDIS_URL="redis://{user}:{password}@{host}:{port}/{db}"
+  export REDIS_READ=true   # enable redis reads (default = REDIS4MARC || false)
+                           # faster reading of triples from pre-populated redis data
+  export REDIS_WRITE=true  # enable redis writes (default = REDIS4MARC || false)
+                           # recent data is updated in redis
+
+HELP
+else
+end
+
+puts "Logging to: #{CONFIG.log_file}"
+marc_files.each do |path|
+  CONFIG.logger.info "Processing: #{path}"
+  marc2ld(path.to_s)
+end
+
+

data/lib/marc2linkeddata/configuration.rb

@@ -0,0 +1,146 @@
+
+module Marc2LinkedData
+
+  class Configuration
+
+    attr_accessor :debug
+
+    attr_accessor :field_auth_loc
+    attr_accessor :field_auth_isni
+    attr_accessor :field_auth_oclc
+    attr_accessor :field_auth_viaf
+
+    attr_accessor :get_isni
+    attr_accessor :get_loc
+    attr_accessor :get_oclc
+    attr_accessor :get_viaf
+    attr_accessor :oclc_auth2works
+
+    attr_accessor :local_loc_user
+    attr_accessor :local_loc_pass
+    attr_accessor :local_loc_url
+
+    attr_accessor :prefixes
+
+    attr_accessor :use_foaf
+    attr_accessor :use_schema
+
+    attr_accessor :redis4marc
+    attr_accessor :redis_read
+    attr_accessor :redis_write
+    attr_accessor :redis
+
+    attr_accessor :log_file
+    attr_accessor :logger
+
+    def initialize
+      @debug = env_boolean('DEBUG')
+
+      # logging
+      log_file = ENV['LOG_FILE'] || 'marc2ld.log'
+      log_file = File.absolute_path log_file
+      @log_file = log_file
+      log_path = File.dirname log_file
+      unless File.directory? log_path
+        # try to create the log directory
+        Dir.mkdir log_path rescue nil
+      end
+      begin
+        log_file = File.new(@log_file, 'w+')
+      rescue
+        log_file = $stderr
+        @log_file = 'STDERR'
+      end
+      @logger = Logger.new(log_file, shift_age = 'monthly')
+      @logger.level = @debug ? Logger::DEBUG : Logger::INFO
+
+      # RDF prefixes
+      @prefixes = {}
+      # Library specific prefixes (use .env file or set shell ENV)
+      @prefixes['lib'] = ENV['LIB_PREFIX'] || 'http://linked-data.stanford.edu/library/'
+      @prefixes['lib_auth'] = "#{prefixes['lib']}authority/"
+      @prefixes['lib_cat']  = "#{prefixes['lib']}catalog/"
+      # Static Prefixes
+      @prefixes['bf'] = 'http://bibframe.org/vocab/'
+      @prefixes['foaf'] = 'http://xmlns.com/foaf/0.1/'
+      @prefixes['isni'] = 'http://www.isni.org/isni/'
+      @prefixes['loc_names'] = 'http://id.loc.gov/authorities/names/'
+      @prefixes['loc_subjects'] = 'http://id.loc.gov/authorities/subjects/'
+      @prefixes['rdf'] = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'
+      @prefixes['rdfs'] = 'http://www.w3.org/2000/01/rdf-schema#'
+      @prefixes['schema'] = 'http://schema.org/'
+      @prefixes['owl'] = 'http://www.w3.org/2002/07/owl#'
+      @prefixes['viaf'] = 'http://viaf.org/viaf/'
+
+      # Authority parse options
+      @field_auth_loc = ENV['FIELD_AUTH_LOC']
+      @field_auth_isni = ENV['FIELD_AUTH_ISNI']
+      @field_auth_oclc = ENV['FIELD_AUTH_OCLC']
+      @field_auth_viaf = ENV['FIELD_AUTH_VIAF']
+
+      @get_isni = env_boolean('GET_ISNI')
+      @get_loc = env_boolean('GET_LOC')
+      @get_viaf = env_boolean('GET_VIAF')
+      @get_oclc = env_boolean('GET_OCLC')
+      @oclc_auth2works = env_boolean('OCLC_AUTH2WORKS')
+
+      # Vocabulary options
+      # foaf:Person or schema:Person or both?
+      @use_foaf = env_boolean('USE_FOAF')
+      @use_schema = env_boolean('USE_SCHEMA') # schema.org
+
+      # Local triple store for LOC authority data,
+      # accessed via an HTTP API with basic authentication.
+      # See downloads at http://id.loc.gov/download/
+      @local_loc_user = ENV['LOCAL_LOC_USER']
+      @local_loc_pass = ENV['LOCAL_LOC_PASS']
+      loc_host = ENV['LOCAL_LOC_HOST']
+      loc_port = ENV['LOCAL_LOC_PORT']
+      loc_path = ENV['LOCAL_LOC_PATH']
+      @local_loc_url = "http://#{loc_host}:#{loc_port}#{loc_path}"
+
+      # Persistence options
+      @redis = nil
+      @redis4marc = env_boolean('REDIS4MARC')
+      if @redis4marc
+        @redis_url = env_boolean('REDIS_URL')
+        @redis_read  = env_boolean('REDIS_READ')
+        @redis_write = env_boolean('REDIS_WRITE')
+        redis_config
+      else
+        @redis_url = nil
+        @redis_read  = false
+        @redis_write = false
+      end
+      # TODO: provide options for triple stores
+    end
+
+    def env_boolean(var)
+      # check if an ENV variable is true, use false as default
+      ENV[var].to_s.upcase == 'TRUE' rescue false
+    end
+
+    def redis_config
+      if @redis4marc
+        # https://github.com/redis/redis-rb
+        # storing objects in redis:
+        #redis.set "foo", [1, 2, 3].to_json
+        #JSON.parse(redis.get("foo"))
+        require 'hiredis'
+        require 'redis'
+        if @redis_url
+          # redis url should be of the form "redis://{user}:{password}@{host}:{port}/{db}"
+          @redis = Redis.new(:url => @redis_url)
+          @redis.ping
+        else
+          # default is 'redis://127.0.0.1:6379/0'
+          @redis = Redis.new
+          @redis.ping
+        end
+      end
+    end
+
+  end
+
+end
+

data/lib/marc2linkeddata/isni.rb

@@ -0,0 +1,23 @@
+require_relative 'resource'
+
+module Marc2LinkedData
+
+  class Isni < Resource
+
+    # Interesting slide presentation about ISNI
+    # http://www.slideshare.net/JaniferGatenby/viaf-and-isni-ifla-2014-0815
+
+    PREFIX = 'http://www.isni.org/isni/'
+
+    def rdf
+      # e.g. 'http://www.isni.org/isni/0000000109311081'
+      return nil if @iri.nil?
+      return @rdf unless @rdf.nil?
+      uri4rdf = @iri.to_s + '.rdf'
+      @rdf = get_rdf(uri4rdf)
+    end
+
+  end
+
+end
+

data/lib/marc2linkeddata/lib_auth.rb

@@ -0,0 +1,17 @@
+require_relative 'resource'
+
+module Marc2LinkedData
+
+  class LibAuth < Resource
+
+    # def rdf
+    #   return nil if @iri.nil?
+    #   return @rdf unless @rdf.nil?
+    #   uri4rdf = @iri.to_s + '.rdf'
+    #   @rdf = get_rdf(uri4rdf)
+    # end
+
+  end
+
+end
+

data/lib/marc2linkeddata/loc.rb

@@ -0,0 +1,91 @@
+require_relative 'resource'
+
+module Marc2LinkedData
+
+  class Loc < Resource
+
+    PREFIX = 'http://id.loc.gov/authorities/'
+    PREFIX_NAMES = "#{PREFIX}names/"
+    PREFIX_SUBJECTS = "#{PREFIX}subjects/"
+
+    # def id
+    #   return nil if @iri.nil?
+    #   @id ||= @iri.basename
+    #   # Could get id from rdf, but that incurs costs for RDF retrieval and parsing etc.
+    #   #oclc_id = '<identifiers:oclcnum>oca04921729</identifiers:oclcnum>'
+    #   #<identifiers:lccn>no 99010609</identifiers:lccn>
+    #   #<identifiers:oclcnum>oca04921729</identifiers:oclcnum>
+    # end
+
+    def rdf
+      return nil if iri.nil?
+      return @rdf unless @rdf.nil?
+      uri4rdf = iri.to_s + '.rdf'
+      @rdf = get_rdf(uri4rdf)
+    end
+
+    def label
+      label_predicate = '<http://www.loc.gov/mads/rdf/v1#authoritativeLabel>'
+      query = SPARQL.parse("SELECT * WHERE { <#{@iri}> #{label_predicate} ?o }")
+      rdf.query(query).first[:o].to_s rescue nil
+    end
+
+    def authority?
+      iri_types.filter {|s| s[:o] == 'http://www.loc.gov/mads/rdf/v1#Authority' }.length > 0
+    end
+
+    def deprecated?
+      iri_types.filter {|s| s[:o] == 'http://www.loc.gov/mads/rdf/v1#DeprecatedAuthority' }.length > 0
+    end
+
+    def conference?
+      iri_types.filter {|s| s[:o] == 'http://www.loc.gov/mads/rdf/v1#ConferenceName' }.length > 0
+    end
+
+    def corporation?
+      iri_types.filter {|s| s[:o] == 'http://www.loc.gov/mads/rdf/v1#CorporateName' }.length > 0
+    end
+
+    def name_title?
+      iri_types.filter {|s| s[:o] == 'http://www.loc.gov/mads/rdf/v1#NameTitle' }.length > 0
+    end
+
+    def person?
+      iri_types.filter {|s| s[:o] == 'http://www.loc.gov/mads/rdf/v1#PersonalName' }.length > 0
+      # iri_types.filter {|s| s[:o] =~ /PersonalName/ }.length > 0
+      # obj = rdf_find_object 'PersonalName'
+      # obj.nil? ? false : true
+    end
+
+    def place?
+      iri_types.filter {|s| s[:o] == 'http://www.loc.gov/mads/rdf/v1#Geographic' }.length > 0
+    end
+
+    def get_oclc_identity
+      # Try to get OCLC URI from LOC ID
+      # http://oclc.org/developer/develop/web-services/worldcat-identities.en.html
+      # e.g. http://www.worldcat.org/identities/lccn-n79044803/
+      # e.g. http://www.worldcat.org/identities/lccn-n79044798/
+      return @oclc_iri unless @oclc_iri.nil?
+      oclc_url = URI.encode('http://www.worldcat.org/identities/lccn-' + id + '/')
+      @oclc_iri = resolve_external_auth(oclc_url)
+      # TODO: OCLC might redirect and then provide a 'fast' URI for obsolete identity records.
+    end
+
+    def get_viaf
+      return @viaf_iri unless @viaf_iri.nil?
+      # Try to get VIAF from LOC sourceID
+      # LOC statement with VIAF URI, e.g.:
+      # s: <http://id.loc.gov/authorities/names/n79046291>
+      # p: <http://www.loc.gov/mads/rdf/v1#hasExactExternalAuthority>
+      # o: <http://viaf.org/viaf/sourceID/LC%7Cn+79046291#skos:Concept> .
+      #return nil unless rdf_valid?
+      #@viaf_iri ||= rdf_find_object 'viaf'
+      viaf_url = URI.encode('http://viaf.org/viaf/sourceID/LC|' + id + '#skos:Concept')
+      @viaf_iri = resolve_external_auth(viaf_url)
+    end
+
+  end
+
+end
+

data/lib/marc2linkeddata/oclc_creative_work.rb

@@ -0,0 +1,44 @@
+require_relative 'oclc_resource'
+
+module Marc2LinkedData
+
+  class OclcCreativeWork < OclcResource
+
+    PREFIX = 'http://www.worldcat.org/oclc/'
+
+    def get_works
+      # assume an exampleOfWork can only ever link to one work?
+      q = query_work(@iri)
+      works = rdf.query(q).collect {|s| s[:o] }
+      if works.empty?
+        # OCLC data is inconsistent in use of 'www.' in IRI, so try again.
+        # The OclcResource coerces @iri so it includes 'www.', so try without it.
+        uri = @iri.to_s.gsub('www.','')
+        q = query_work(uri)
+        works = rdf.query(q).collect {|s| s[:o] }
+      end
+      if works.empty?
+        # Keep the 'www.', cast the ID to an integer.
+        uri = @iri.to_s.gsub(id, id.to_i.to_s)
+        q = query_work(uri)
+        works = rdf.query(q).collect {|s| s[:o] }
+      end
+      if works.empty?
+        # Remove the 'www.' AND cast the ID to an integer.
+        uri = @iri.to_s.gsub('www.','').gsub(id, id.to_i.to_s)
+        q = query_work(uri)
+        works = rdf.query(q).collect {|s| s[:o] }
+      end
+      works
+    end
+
+    def query_work(uri)
+      SPARQL.parse("SELECT * WHERE { <#{uri}> <http://schema.org/exampleOfWork> ?o }")
+    end
+
+    # TODO: get ISBN?
+
+  end
+
+end
+

data/lib/marc2linkeddata/oclc_identity.rb

@@ -0,0 +1,46 @@
+require_relative 'resource'
+
+module Marc2LinkedData
+
+  class OclcIdentity < Resource
+
+    PREFIX = 'http://www.worldcat.org/identities/'
+
+    def rdf
+      # e.g. 'http://www.worldcat.org/identities/lccn-n79044803/'
+      # the html returned contains RDFa data
+      return nil if @iri.nil?
+      return @rdf unless @rdf.nil?
+      uri4rdf = @iri.to_s
+      uri4rdf += '/' unless uri4rdf.end_with? '/'
+      @rdf = get_rdf(uri4rdf)
+    end
+
+    # def get_xml
+    #   begin
+    #     return @xml unless @xml.nil?
+    #     http = Net::HTTP.new @iri.host
+    #     resp = http.get(@iri.path, {'Accept' => 'application/xml'})
+    #     case resp.code
+    #       when '301','302','303'
+    #         #301 Moved Permanently; 302 Moved Temporarily; 303 See Other
+    #         resp = http.get(resp['location'], {'Accept' => 'application/xml'})
+    #     end
+    #     if resp.code != '200'
+    #       raise
+    #     end
+    #     @xml = resp.body
+    #   rescue
+    #     puts 'ERROR: Failed to request OCLC identity xml.'
+    #   end
+    # end
+
+    def creative_works
+      q = SPARQL.parse('SELECT * WHERE { ?oclcWork a <http://schema.org/CreativeWork> }')
+      rdf.query(q).collect {|s| s[:oclcWork] }
+    end
+
+  end
+
+end
+

data/lib/marc2linkeddata/oclc_resource.rb

@@ -0,0 +1,79 @@
+require_relative 'resource'
+
+module Marc2LinkedData
+
+  class OclcResource < Resource
+
+    PREFIX = 'http://www.worldcat.org/oclc/'
+
+    def initialize(uri=nil)
+      # Ensure the OCLC IRI contains 'www' in the host name.
+      unless uri =~ /www\./
+        uri = uri.to_s.gsub('worldcat.org','www.worldcat.org')
+      end
+      super(uri)
+    end
+
+    def rdf
+      # e.g. 'http://worldcat.org/oclc/004957186'
+      # also 'http://www.worldcat.org/oclc/004957186'
+      return nil if @iri.nil?
+      return @rdf unless @rdf.nil?
+      uri4rdf = @iri.to_s
+      uri4rdf += '.rdf' unless uri4rdf.end_with? '.rdf'
+      @rdf = get_rdf(uri4rdf)
+    end
+
+    def book?
+      iri_types.filter {|s| s[:o] == 'http://schema.org/Book' }.length > 0
+    end
+
+    def creator?(uri)
+      creators.include? RDF::URI.new(uri)
+    end
+
+    def contributor?(uri)
+      contributors.include? RDF::URI.new(uri)
+    end
+
+    def editor?(uri)
+      editors.include? RDF::URI.new(uri)
+    end
+
+    def media_object?
+      iri_types.filter {|s| s[:o] == 'http://schema.org/MediaObject' }.length > 0
+    end
+
+    def about
+      q = SPARQL.parse('SELECT * WHERE { ?s <http://schema.org/about> ?o }')
+      rdf.query(q)
+    end
+
+    def creators
+      q = SPARQL.parse("SELECT * WHERE { <#{@iri}> <http://schema.org/creator> ?o }")
+      rdf.query(q).collect {|s| s[:o] }
+    end
+
+    def contributors
+      q = SPARQL.parse("SELECT * WHERE { <#{@iri}> <http://schema.org/contributor> ?o }")
+      rdf.query(q).collect {|s| s[:o] }
+    end
+
+    def editors
+      q = SPARQL.parse("SELECT * WHERE { <#{@iri}> <http://schema.org/editor> ?o }")
+      rdf.query(q).collect {|s| s[:o] }
+    end
+
+    def publishers
+      q = SPARQL.parse("SELECT * WHERE { <#{@iri}> <http://schema.org/publisher> ?o }")
+      rdf.query(q).collect {|s| s[:o] }
+    end
+
+    def isbns
+      q = SPARQL.parse("SELECT * WHERE { <#{@iri}> <http://schema.org/isbn> ?o }")
+      rdf.query(q).collect {|s| s[:o] }
+    end
+  end
+
+end
+

data/lib/marc2linkeddata/oclc_work.rb

@@ -0,0 +1,19 @@
+require_relative 'oclc_resource'
+
+module Marc2LinkedData
+
+  class OclcWork < OclcResource
+
+    # OCLC is inconsistent with use of 'www' in URIs
+    #PREFIX = 'http://www.worldcat.org/entity/work/id/'
+    PREFIX = 'http://worldcat.org/entity/work/id/'
+
+    def example_works
+      q = SPARQL.parse("SELECT * WHERE { <#{@iri}> <http://schema.org/workExample> ?o }")
+      rdf.query(q).collect {|s| s[:o] }
+    end
+
+  end
+
+end
+