marc2linkeddata 0.0.7 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f54e31c819d0978629746c13c88690d5b2f1f392
-  data.tar.gz: 5325910c217a5e1bd3d45bb0a1f0bdb3b48fa11e
+  metadata.gz: f9539a926fe1d42b3827b55c4e44245859f0c1e2
+  data.tar.gz: 048690d375535f681bf518af81639d6b6dca4af4
 SHA512:
-  metadata.gz: 2565a8648875bc22f3b62ccd7cd1aad47ce843d388a93f6e4e363f7bab77644abbf56a1a70f73de9d6ac9aa9be3857578b62c87e5ebc2d4e50b157afaea7c7ae
-  data.tar.gz: a8ba98f07795a0c97f4bd00d9c431ea7a4dd15d39952f1cdbc56a4136d4b6e60f2bb0a6d05cb6f3affe5d5eecd03dc6752b4093dcb8b149f61d36583911af610
+  metadata.gz: 7d42f2629e02882f94e60ceb1fe8503766d88c2cd108b0e57fb87d8704663f6a693667d098832d4c793e5e3ff285b04b3541d297d989529e63782dfbae5289a8
+  data.tar.gz: fa4fe996e40a4de2cb0bffab698dc76c843612ab59055c498b0a99af4f6d50a05156c0e715bcbfd074643e04ff0578687e555e3753eeb6990417f2f246ed11ab

data/.env_example

@@ -1,46 +1,71 @@
 # https://github.com/bkeepers/dotenv is used for
 # default configuration options.  The values in
 # this file do not replace existing values in
-# the shell ENV.
+# the shell ENV, but these settings will be in
+# effect when the shell ENV doesn't contain them.
+# To add these settings to the shell ENV, use
+# $ source .env
+# To override individual settings in this file, just
+# set them in the shell ENV before running a script.
 
 # Uncomment and set values as required.  See used settings in
 # lib/marc2linkeddata/configuration.rb
 
-DEBUG: false
+export DEBUG=false
+
+export LOG_FILE='marc2ld.log'
+export LIB_PREFIX=http://linked-data.example.org/library/
+
+# Without any options enabled to GET_* via HTTP, the process
+# is largely file IO bound, rather than CPU bound.  In this
+# case, threading can decrease performance.  However, when any
+# GET_* options are enabled for retrieval of RDF over HTTP,
+# threading may improve performance.
+export THREADS=true
+export THREAD_LIMIT=25
 
 # Authority record field numbers for useful link data
-FIELD_AUTH_LOC:  920
-FIELD_AUTH_VIAF: 921
-FIELD_AUTH_ISNI: 922
-FIELD_AUTH_OCLC: 035
+export FIELD_AUTH_LOC=920
+export FIELD_AUTH_VIAF=921
+export FIELD_AUTH_ISNI=922
+export FIELD_AUTH_OCLC=035
 
 # Options for retrieving linked data to resolve and enhance data.
-# Set all false for the quickest translation.
-GET_ISNI: false
-GET_LOC: true    # currently required, should be optional
-GET_OCLC: false
-GET_VIAF: false
+# Set all false for the quickest translation.  Note that having an
+# LOC identifier (MARC record number may be sufficient), is usually
+# a prelude to getting additional linked data.  If any of these are
+# enabled, it may require GET_LOC=true to have greater success.
+export GET_LOC=false
+export GET_ISNI=false
+export GET_OCLC=false
+export GET_VIAF=false
 
 # Using OCLC identity, retrieve RDF for creative works?
 # Only works when GET_OCLC==true; it can slow processing significantly.
-OCLC_AUTH2WORKS: false
-
-LOG_FILE: 'marc2ld.log'
-
-LIB_PREFIX: http://linked-data.example.org/library/
+export OCLC_AUTH2WORKS=false
 
 # Use FOAF or SCHEMA or both
-USE_FOAF: false
-USE_SCHEMA: true
+export USE_FOAF=false
+export USE_SCHEMA=true
 
 # Local triple store for LOC authority data,
 # accessed via an HTTP API with basic authentication.
 # See downloads at http://id.loc.gov/download/
-LOCAL_LOC_USER: 'sparqlUser'
-LOCAL_LOC_PASS: 'sparqlPass'
-LOCAL_LOC_HOST: 'dev-sparql.example.org'
-LOCAL_LOC_PORT: '80'
-LOCAL_LOC_PATH: '/sparql?'
+export LOCAL_LOC_USER='sparqlUser'
+export LOCAL_LOC_PASS='sparqlPass'
+export LOCAL_LOC_HOST='dev-sparql.example.org'
+export LOCAL_LOC_PORT='80'
+export LOCAL_LOC_PATH='/sparql?'
+
+# SUL-CAP resources
+# May require ssh port forwarding, e.g.:
+# ssh ${USER}@cap-mysql-host.example.com -L 3308:localhost:3306 -N &
+export SUL_CAP_ENABLED=false
+export SUL_CAP_DB_HOST=localhost
+export SUL_CAP_DB_PORT=3306
+export SUL_CAP_DB_USER=capUser
+export SUL_CAP_DB_PASSWORD=capPass
+export SUL_CAP_DB_DATABASE=cap
 
 # Redis Persistence - based on https://github.com/redis/redis-rb
 # - essential options:
@@ -52,11 +77,11 @@ LOCAL_LOC_PATH: '/sparql?'
 #                           # faster reading of triples from pre-populated redis data
 #   export REDIS_WRITE=true # enable redis writes (default = REDIS4MARC || false)
 #                           # current data is updated in redis
-REDIS4MARC: false
+export REDIS4MARC=false
 # Uncomment these options to disable read or write (independently)
-#REDIS_READ: false
-#REDIS_WRITE: false
+#export REDIS_READ=false
+#export REDIS_WRITE=false
 # Leave commented to use default redis configs on localhost
-#REDIS_URL: localhost
+#export REDIS_URL=localhost
 
 

data/README.md

@@ -3,13 +3,44 @@ marc2linkeddata
 ===============
 
 Utilities for translating MARC21 into linked data.  The project has
-focused on authority records (as of Feb, 2015).
+focused on authority records (as of 2015).
+
+It has config options that can be enabled to increase the amount of data retrieved.
+Without any HTTP options enabled, using only data in the MARC record, it can
+translate 100,000 authority records in about 5-6 min on a current laptop system.
+File IO is the most expensive operation in this mode, so it helps to have a solid
+state drive or something with high IO performance.
+
+The current output is to the file system, but it should be easy to incorporate
+and configure alternatives by using the RDF.rb facilities for connecting to a
+repository.  A minor attempt was explored to use redis for caching, but that
+exploration hasn't matured much, mainly because there is no 'cache-expiry' data
+yet and because it would be better to use an RDF.rb extension of some
+kind (for redis, mongodb, etc) or to use a triple store/solr platform.
+
+With HTTP/RDF retrieval options enabled, it can take a lot longer (days) and the
+providers may not be very happy about a barrage of requests.
+
+Note that it runs a lot slower on jruby-9.0.0.0-pre1 than MRI 2.2.0, whether threads
+are enabled or not.  It raises exceptions on jruby-1.7.9, related to ruby
+language support (such as Array#delete_if).
+
+TODO: A significant problem to solve is effective caching or mirrors for linked data.
+The retrieval should inspect any HTTP cache headers that might be available and
+adding PROVO to the linked-data graph generated for each record.
+
+TODO: Provide system platform options, to dockerize the application and make it easier
+for automatic horizontal scaling.  Consider https://www.packer.io/intro/index.html
 
 Optional Dependencies
 
-  - http://redis.io/
   - http://4store.org/
+  - http://www.mongodb.org/
+  - http://redis.io/
   - see notes below
+  - see also:
+    - http://marmotta.apache.org
+    - http://stardog.com
 
 Install
 

data/bin/marcAuthority2LD

@@ -4,6 +4,17 @@ require 'marc2linkeddata'
 
 CONFIG = Marc2LinkedData.configuration
 
+def stack_trace(e, record)
+  $stderr.write "\n"
+  $stderr.write "ERROR\n"
+  $stderr.write e.message
+  $stderr.write e.backtrace
+  $stderr.write "\n"
+  $stderr.write record.to_s
+  $stderr.write "\n"
+end
+
+# Count all the records in the MARC file.
 def marc_auth_count(marc_file)
   auth_records = 0
   until marc_file.eof?
@@ -12,11 +23,7 @@ def marc_auth_count(marc_file)
       marc_file.seek(leader[:length], IO::SEEK_CUR)
       auth_records += 1 if leader[:type] == 'z'
     rescue => e
-      puts
-      puts 'ERROR'
-      puts e.message
-      puts e.backtrace
-      puts
+      stack_trace(e, record)
       binding.pry if CONFIG.debug
     end
   end
@@ -24,7 +31,76 @@ def marc_auth_count(marc_file)
   auth_records
 end
 
-def marc2ld(marc_filename)
+# Memory intensive loading of all authority records in the MARC file.
+def marc_authority_records(marc_filename)
+  puts "Reading records from: #{marc_filename}"
+  marc_file = File.open(marc_filename,'r')
+  auth_count = 0
+  auth_records = []
+  until marc_file.eof?
+    begin
+      leader = Marc2LinkedData::ParseMarcAuthority::parse_leader(marc_file)
+      raw = marc_file.read(leader[:length])
+      if leader[:type] == 'z'
+        record = MARC::Reader.decode(raw)
+        auth_records << record
+        auth_count += 1
+        $stdout.printf "\b\b\b\b\b\b" if auth_count > 1
+        $stdout.printf '%06d', auth_count
+      end
+    rescue => e
+      stack_trace(e, record)
+      binding.pry if CONFIG.debug
+    end
+  end
+  marc_file.close
+  $stdout.write "\n"
+  auth_records
+end
+
+
+def auth_record_cache(auth)
+
+  # auth_id = "auth:#{auth.get_id}"
+  # triples = nil
+  # # TODO: enable additional persistence options
+  # # Use data already in redis (if enabled)
+  # triples = CONFIG.redis.get(auth_id) if CONFIG.redis_read
+  # if triples.nil?
+  #   triples = auth.to_ttl  # generate new triples
+  #   # Update redis (if enabled) for triples not read from redis
+  #   CONFIG.redis.set(auth_id, triples) if CONFIG.redis_write
+  # end
+
+end
+
+
+
+def marc_record2turtle(record, output_path=nil)
+  begin
+    # ParseMarcAuthority is a lazy parser, so
+    # init only assigns record to an instance var.
+    auth = Marc2LinkedData::ParseMarcAuthority.new(record)
+    auth_record_cache(auth)
+    triples = auth.to_ttl.lines
+    binding.pry if (CONFIG.debug && triples.empty?)
+    triples.delete_if {|l| l.chomp.empty? }
+    # Output the triples to a turtle file.
+    ld_filename = File.join(output_path, "auth_#{auth.get_id}.ttl")
+    CONFIG.logger.info "Writing triples in turtle to #{ld_filename}"
+    ld_file = File.open(ld_filename,'w')
+    ld_file.write(triples.join)
+    ld_file.flush
+    ld_file.close
+  rescue => e
+    stack_trace(e, record)
+    binding.pry if CONFIG.debug
+  end
+end
+
+
+
+def marc_file2ld(marc_filename)
   ld_filename = marc_filename.gsub('.mrc','.ttl')
   puts "Translating: #{marc_filename} to #{ld_filename}"
   ld_file = File.open(ld_filename,'w')
@@ -44,17 +120,7 @@ def marc2ld(marc_filename)
         # init only assigns record to an instance var.
         auth = Marc2LinkedData::ParseMarcAuthority.new(record)
         auth_count += 1
-        # auth_id = "auth:#{auth.get_id}"
-        # triples = nil
-        # # TODO: enable additional persistence options
-        # # Use data already in redis (if enabled)
-        # triples = CONFIG.redis.get(auth_id) if CONFIG.redis_read
-        # if triples.nil?
-        #   triples = auth.to_ttl  # generate new triples
-        #   # Update redis (if enabled) for triples not read from redis
-        #   CONFIG.redis.set(auth_id, triples) if CONFIG.redis_write
-        # end
-
+        auth_record_cache(auth)
         triples = auth.to_ttl.lines
         binding.pry if (CONFIG.debug && triples.empty?)
         triples.delete_if {|l| l.chomp.empty? }
@@ -63,12 +129,7 @@ def marc2ld(marc_filename)
         ld_file.flush
       end
     rescue => e
-      puts
-      puts 'ERROR'
-      puts e.message
-      puts e.backtrace
-      puts record.to_s
-      puts
+      stack_trace(e, record)
       binding.pry if CONFIG.debug
     end
   end
@@ -77,38 +138,68 @@ def marc2ld(marc_filename)
   ld_file.close
 end
 
+def thread_wait(threads)
+  threads.each{|t| t.join}
+  threads.delete_if {|t| t.status == false}
+  threads.delete_if {|t| t.status.nil? }
+end
+
+
+# ---------------------------------------------------------------------
+# MAIN
+
+
 marc_files = []
 ARGV.each do |filename|
   path = Pathname(filename)
   marc_files.push(path) if path.exist?
 end
 if marc_files.empty?
+  script_name = File.basename(__FILE__)
+  script_path = File.dirname(__FILE__)
+  example_env_file = File.absolute_path(File.join(script_path,'..','.env_example'))
   puts <<HELP
-#{__FILE__} marc_authority_file1.mrc [ marc_authority_file2.mrc .. marc_authority_fileN.mrc ]
-
-Output is RDF triples in a turtle file (.ttl) for every input .mrc file.
-Optional persistence services can be controlled by environment variables.
-
-Redis Persistence - based on https://github.com/redis/redis-rb
-- essential options:
-  export REDIS4MARC=true # enable redis persistence (default = false)
-- supplementary options:
-  Set the REDIS_URL for a custom redis configuration.
-  export REDIS_URL="redis://{user}:{password}@{host}:{port}/{db}"
-  export REDIS_READ=true   # enable redis reads (default = REDIS4MARC || false)
-                           # faster reading of triples from pre-populated redis data
-  export REDIS_WRITE=true  # enable redis writes (default = REDIS4MARC || false)
-                           # recent data is updated in redis
+#{script_name} marc_authority_file1.mrc [ marc_authority_file2.mrc .. marc_authority_fileN.mrc ]
+
+Output is RDF triples in turtle files (.ttl). The files are output into a
+directory created in the same path as the .mrc file,
+with one .ttl file for each record.
+
+Optional configuration can be set in environment variables. A '.env' file can be
+created in the path where this utility is run and this utility will use it. See
+comments and settings in the example file at:
+#{example_env_file}
 
 HELP
   exit!
-else
 end
 
 puts "Logging to: #{CONFIG.log_file}"
 marc_files.each do |path|
   CONFIG.logger.info "Processing: #{path}"
-  marc2ld(path.to_s)
+  # marc_file2ld(path.to_s)
+  output_dir = path.basename.to_s.gsub('.mrc','').gsub('.','_') + '_turtle'
+  output_path = File.join(path.dirname.to_s, output_dir)
+  Dir.mkdir(output_path, 0775) unless File.directory? output_path
+  auth_records = marc_authority_records(path.to_s)
+  progress = ProgressBar.create(:total => auth_records.length, :format => '%a %f |%b>>%i| %P%% %t')
+  if CONFIG.threads
+    threads = []
+    auth_records.each do |r|
+      thread_wait(threads) while threads.length >= CONFIG.thread_limit
+      t = Thread.new { marc_record2turtle(r, output_path) }
+      t.abort_on_exception = true
+      threads << t
+      progress.increment  # increment progress although thread is still running
+    end
+    # Ensure any remaining threads complete
+    threads.each{|t| t.join}
+  else
+    auth_records.each do |r|
+      marc_record2turtle(r, output_path)
+      progress.increment
+    end
+  end
 end
 
 

data/bin/run_test_data.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+export DEBUG=false
+
+# Runs a lot slower on jruby, even with threading enabled.
+export JRUBY_OPTS=-J-Xmx2g
+
+export THREAD=false
+export GET_LOC=false  # if this is true, be prepared to wait a very long time!
+
+export LOG_FILE='./log/marc2ld.log'
+export LIB_PREFIX='http://www.linked-data.org/library/'
+
+# Additional config options should be in .env;
+# the .env values will not replace those above.
+if [ ! -s .env ]; then
+    cp -u .env_example .env
+fi
+
+SCRIPT_FILE='.binstubs/marcAuthority2LD'
+if [ ! -s ${SCRIPT_FILE} ]; then
+    echo "Cannot locate script: $SCRIPT_FILE"
+    exit 1
+fi
+
+AUTH_FILE="./data/auth.mrc"
+AUTH_PATH="./data/auth_turtle/"
+if [ ! -s ${AUTH_FILE} ]; then
+    echo "Place a MARC21 authority file into: $AUTH_FILE"
+    exit 1
+fi
+
+${SCRIPT_FILE} ${AUTH_FILE}
+
+DATA_LOG_FILE="./log/run_test_data.log"
+echo -e "\n\n" > ${DATA_LOG_FILE}
+
+echo -e "Output file count should be 100000:" >> ${DATA_LOG_FILE}
+find ${AUTH_PATH} -type f | wc -l  >> ${DATA_LOG_FILE}
+echo -e "\n\n" > ${DATA_LOG_FILE}
+
+# count all the different types of authority files
+echo -e "Different types of authority records:\n" >> ${DATA_LOG_FILE}
+find ${AUTH_PATH} -type f | xargs grep 'linked-data' | sed -e 's/^.*> a/a/' | sort -u >> ${DATA_LOG_FILE}
+echo -e "\n\n" > ${DATA_LOG_FILE}
+
+echo -e "Count for 'Person' authority records:" >> ${DATA_LOG_FILE}
+find ${AUTH_PATH} -type f | xargs grep 'linked-data' | grep -c -F 'Person' >> ${DATA_LOG_FILE}
+echo -e "Count for 'Organization' authority records:" >> ${DATA_LOG_FILE}
+find ${AUTH_PATH} -type f | xargs grep 'linked-data' | grep -c -F 'Organization' >> ${DATA_LOG_FILE}
+echo -e "Count for 'Place' authority records:" >> ${DATA_LOG_FILE}
+find ${AUTH_PATH} -type f | xargs grep 'linked-data' | grep -c -F 'Place' >> ${DATA_LOG_FILE}
+echo -e "Count for 'event' authority records:" >> ${DATA_LOG_FILE}
+find ${AUTH_PATH} -type f | xargs grep 'linked-data' | grep -c -F 'event' >> ${DATA_LOG_FILE}
+echo -e "Count for 'v1#NameTitle' authority records:" >> ${DATA_LOG_FILE}
+find ${AUTH_PATH} -type f | xargs grep 'linked-data' | grep -c -F 'v1#NameTitle' >> ${DATA_LOG_FILE}
+echo -e "Count for 'v1#Title' authority records:" >> ${DATA_LOG_FILE}
+find ${AUTH_PATH} -type f | xargs grep 'linked-data' | grep -c -F 'v1#Title' >> ${DATA_LOG_FILE}
+
+# # check the syntax of the output files
+# echo -e "\n\n\n" >> ${DATA_LOG_FILE}
+# for f in $(find ${AUTH_PATH} -type f); do
+#      rapper -c -i turtle $f >> ${DATA_LOG_FILE} 2>&1
+# done
+
+# cleanup
+rm -rf ${AUTH_PATH}
+