cap-vivo-mapper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e938fd82abdc61cc4cf0a92494f51fc7e74d485d
+  data.tar.gz: 1887b0969c45d7e05b394f1da3b518f6f9bc9c76
+SHA512:
+  metadata.gz: 1491afbafa6105146cf1c556038062e9f6a7ec09e82db208ee25ad1d0be320748b1cc7ce8682b271558fe2d274a4b4ca01d145fa0ba47297d47345e74965d8e0
+  data.tar.gz: fddf639dfebae49a33d939c2d8e9deee62605985a5b2e6a25fc1066ffc94c6a6cb61bad1765cf5225174032ffe9ef70bcf840999feb217bb23c5e503acee5fb3

data/.env_example

@@ -0,0 +1,37 @@
+# https://github.com/bkeepers/dotenv is used for
+# default configuration options.  The values in
+# this file do not replace existing values in
+# the shell ENV.
+
+# Uncomment and set values as required.  See used settings in
+# lib/*/configuration.rb
+
+export DEBUG=false
+
+export CAP_VIVO_LOG_FILE='log/cap_vivo_mapper.log'
+
+# false = convert all CAP profiles into VIVO linked data, replacing any
+#         existing VIVO records
+# true  = do not replace existing VIVO records, only convert CAP profiles
+#         that are not already VIVO records
+export CAP_VIVO_REPLACE=false
+
+# Configure the client service
+export CAP_API_URL='https://cap.example.com'
+export CAP_API_ORGS='/api/cap/v1/orgs'
+export CAP_API_PROFILES='/api/profiles/v1'
+export CAP_API_SCHEMA='/api/cap/v1/schemas'
+export CAP_API_SEARCH='/api/cap/v1/search'
+export CAP_API_SEARCH_AC='/api/cap/v1/search/autocomplete'
+export CAP_API_SEARCH_KW='/api/cap/v1/search/keyword'
+export CAP_API_LOG_FILE='log/cap_vivo_mapper.log'
+
+# Parameters for client authentication
+export CAP_TOKEN_URI='https://authz.example.com/oauth/token'
+export CAP_TOKEN_USER=user
+export CAP_TOKEN_PASS=pass
+export CAP_TOKEN_ACCESS=xyz.etc
+
+# Parameters for CAP and VIVO data persistence
+export CAP_REPO_4STORE='http://localhost:9000'
+export CAP_REPO_MONGO='mongodb://127.0.0.1:27017/cap'

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cap-vivo-mapper.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright 2015 The Board of Trustees of the Leland Stanford Junior University.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,110 @@
+# Cap::Vivo::Mapper
+
+This utility maps Stanford CAP profiles to VIVO.
+
+## Initial Objectives
+
+* Identification of how CAP overlaps with the basics of the VIVO-ISF model
+  * specifically as relates to People and their relationships
+  * e.g. the LODE and eagle-i views, plus docs and examples on the wiki
+* A test case transformation
+  * A document mapping CAP person data to VIVO-ISF
+  * A json transform utility to implement the mapping
+  * Retrieving CAP profile data from the CAP API
+* Also investigate CAP publication data
+  * consider mapping to both VIVO-ISF and simple BibFrame
+
+### Stanford CAP Resources
+
+- https://cap.stanford.edu/cap-api/console
+
+### VIVO Resources
+
+- https://wiki.duraspace.org/display/VIVO/VIVO
+- https://wiki.duraspace.org/display/VIVO/Major+concepts+in+VIVO+to+get+you+started
+- https://wiki.duraspace.org/display/VIVO/VIVO-ISF+Ontology
+- http://www.vivoweb.org/download
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'cap-vivo-mapper'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install cap-vivo-mapper
+
+## Usage
+
+### Setup
+
+#### MongoDB
+
+```sh
+# The following worked on an Ubuntu desktop system
+sudo apt-get install mongodb
+```
+
+#### 4store
+
+```sh
+# The following worked on an Ubuntu desktop system
+sudo apt-get install 4store
+sudo 4store status
+sudo service 4store stop
+sudo service 4store status
+# Only setup the backend once (it erases existing data)
+sudo 4s-backend-setup cap_vivo
+sudo 4s-backend cap_vivo
+sudo 4s-httpd -h # describes the options used below
+sudo 4s-httpd -p 9000 -U -s -1 cap_vivo
+```
+
+4store should be running a SPARQL server on the `cap_vivo` knowledge base; take a look at http://localhost:9000/status/.
+
+### Configure and Run Conversion
+
+Use the example configuration in
+https://github.com/sul-dlss/cap-vivo-mapper/blob/master/.env_example
+
+```sh
+mkdir -p ~/tmp/cap_vivo/log
+cd ~/tmp/cap_vivo
+project='https://raw.githubusercontent.com/sul-dlss/cap-vivo-mapper'
+wget ${project}/master/.env_example
+cp .env_example .env
+vim .env  # hopefully this file is self explanatory
+# If it's not already installed, install the the gem.
+gem install cap-vivo-mapper
+# Run it overnight, unless you have a high bandwidth connection to the
+# CAP API and a fast system.  So, watch it for any immediate failures;
+# if it's running, then leave it overnight.  The expected runtime is on
+# the order of hours.
+cap2vivo
+```
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/sul-dlss/cap-vivo-mapper.
+
+## License
+
+Copyright 2015 The Board of Trustees of the Leland Stanford Junior University.
+
+The gem is available as open source under the terms of the [Apache 2 License](http://www.apache.org/licenses/LICENSE-2.0).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/cap2vivo

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'bundler/setup'
+
+require 'pry'
+require 'cap'
+require 'cap/client'
+require 'cap/vivo'
+
+# init configs
+config = Cap.configuration
+
+# client will load profile data from a local repo; if it is empty, it
+# can be populated from the CAP API using `client.get_profiles`.  The
+# profile data is available in `client.profiles`.
+client = Cap::Client::Client.new
+client.get_profiles
+
+# Convert all the CAP API profiles into VIVO linked data and
+# store it in the CAP_REPO_4STORE triple store.
+ids = client.profile_ids
+if config.rdf_replace
+  puts 'Replacing all VIVO linked data'
+  config.rdf_repo.clear
+else
+  puts 'Updating VIVO linked data'
+  q = 'SELECT ?person WHERE { ?person a <http://xmlns.com/foaf/0.1/Person> }'
+  r = config.rdf_repo.client.query(q)
+  vivo_uris = r.map {|s| s[:person]}
+  vivo_ids = vivo_uris.map {|uri| uri.to_s.split('/').last.to_i }
+  ids.delete_if {|id| vivo_ids.include? id }
+end
+puts "Processing #{ids.length} CAP profiles"
+count = 0
+start = Time.now.to_i
+ids.each do |id|
+  count += 1
+  profile = client.profile(id)
+  begin
+    mapper = Cap::Vivo::Mapper.new profile
+    mapper.create_vivo
+    mapper.save
+    if count % 25 == 0
+      delay = Time.now.to_i - start
+      printf "\nmapped %4d of %d (%4d sec)\n", count, ids.length, delay.to_s
+    else
+      if count == 1
+        printf "Mapping %4d profiles:\n", ids.length
+      end
+      printf '.'
+    end
+  rescue => e
+    delay = Time.now.to_i - start
+    puts
+    printf "FAILED profileId %d (%d of %d)\n", id, count, ids.length
+    puts e.message
+    puts
+  end
+end
+puts
+
+at_exit {
+  config.cap_repo.close if config.cap_repo.is_a? Daybreak::DB
+  client = nil
+  config = nil
+  sleep 2 # give it a chance to close
+}

data/cap-vivo-mapper.gemspec

@@ -0,0 +1,51 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cap/vivo/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "cap-vivo-mapper"
+  spec.version       = Cap::Vivo::VERSION
+  spec.authors       = ["Darren L. Weber, Ph.D."]
+  spec.email         = ["darren.weber@stanford.edu"]
+
+  spec.summary       = %q{This utility maps Stanford CAP profiles to VIVO.}
+  spec.description   = %q{This utility maps Stanford CAP profiles to VIVO.}
+  spec.homepage      = 'https://github.com/sul-dlss/cap-vivo-mapper'
+  spec.licenses      = ['Apache-2.0']
+
+  spec.add_dependency 'dotenv'
+
+  spec.add_dependency 'daybreak' # memory mapped file db
+  spec.add_dependency 'mongo'
+
+  spec.add_dependency 'linkeddata'
+  spec.add_dependency 'rdf-4store'
+
+  spec.add_dependency 'faraday'
+  spec.add_dependency 'faraday_middleware'
+
+  # Use pry for console and debug config
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry-doc'
+
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'vcr'
+  spec.add_development_dependency 'webmock'
+
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'guard'
+  spec.add_development_dependency 'guard-ctags-bundler'
+
+  git_files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  bin_files = %w(bin/console bin/ctags.rb bin/setup bin/test.rb)
+  dot_files = %w(.gitignore .travis.yml log/.gitignore)
+
+  spec.files         = git_files - (bin_files + dot_files)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+end

data/lib/cap.rb

@@ -0,0 +1,32 @@
+require 'dotenv'
+Dotenv.load
+
+require 'linkeddata'
+require 'rdf/4store'
+require 'daybreak'
+require 'mongo'
+require_relative 'cap/configuration'
+
+# This is a utility working with Stanford CAP and VIVO data mappings.
+# https://github.com/sul-dlss/cap-vivo-mapper
+module Cap
+
+  # Configuration at the module level, see
+  # http://brandonhilkert.com/blog/ruby-gem-configuration-patterns/
+  class << self
+    attr_writer :configuration
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.reset
+    @configuration = Configuration.new
+  end
+
+  def self.configure
+    yield(configuration)
+  end
+
+end

data/lib/cap/client.rb

@@ -0,0 +1,31 @@
+require 'dotenv'
+Dotenv.load
+
+require_relative 'client/configuration'
+require_relative 'client/cap_client'
+
+# This is a utility working with Stanford CAP.
+# https://github.com/sul-dlss/cap-vivo-mapper
+module Cap
+  module Client
+
+    # Configuration at the module level, see
+    # http://brandonhilkert.com/blog/ruby-gem-configuration-patterns/
+    class << self
+      attr_writer :configuration
+    end
+
+    def self.configuration
+      @configuration ||= Configuration.new
+    end
+
+    def self.reset
+      @configuration = Configuration.new
+    end
+
+    def self.configure
+      yield(configuration)
+    end
+
+  end
+end

data/lib/cap/client/cap_client.rb

@@ -0,0 +1,336 @@
+module Cap
+  module Client
+
+    require 'faraday'
+    require 'faraday_middleware'
+
+    # CAP Public Website  https://profiles.stanford.edu
+    # Profiles API        https://api.stanford.edu/profiles/v1
+    # Orgs API            https://api.stanford.edu/cap/v1/orgs
+    # Search API          https://api.stanford.edu/cap/v1/search
+    # Developer's API     https://cap.stanford.edu/cap-api/console
+
+    class Client
+
+      JSON_CONTENT = 'application/json'
+      BSON_MAX = 16777216
+
+      attr_reader :config
+      attr_reader :cap_api
+      attr_reader :profiles
+
+      # Initialize a new client
+      def initialize
+        @config = Cap::Client.configuration
+        if Cap.configuration.cap_repo.is_a? Daybreak::DB
+          @profiles = Cap.configuration.cap_repo
+        elsif Cap.configuration.cap_repo.is_a? Mongo::Client
+          @profiles = Cap.configuration.cap_repo[:profiles]
+          @presentations = Cap.configuration.cap_repo[:presentations]
+          @publications = Cap.configuration.cap_repo[:publications]
+          @processed = Cap.configuration.cap_repo[:processed]
+        end
+        # CAP API
+        @cap_uri = 'https://api.stanford.edu'
+        @cap_profiles = '/profiles/v1'
+        @cap_orgs     = '/cap/v1/orgs'
+        @cap_search   = '/cap/v1/search'
+        @cap_api = Faraday.new(url: @cap_uri) do |f|
+          # f.use FaradayMiddleware::FollowRedirects, limit: 3
+          # f.use Faraday::Response::RaiseError # raise exceptions on 40x, 50x
+          # f.request :logger, @config.logger
+          f.request :json
+          f.response :json, :content_type => JSON_CONTENT
+          f.adapter Faraday.default_adapter
+        end
+        @cap_api.options.timeout = 90
+        @cap_api.options.open_timeout = 10
+        @cap_api.headers.merge!(json_payloads)
+        # Authentication
+        auth_uri = 'https://authz.stanford.edu/oauth/token'
+        @auth = Faraday.new(url: auth_uri) do |f|
+          f.request  :url_encoded
+          f.response :json, :content_type => JSON_CONTENT
+          f.adapter  Faraday.default_adapter
+        end
+        @auth.options.timeout = 30
+        @auth.options.open_timeout = 10
+        @auth.headers.merge!(json_payloads)
+      end
+
+      # Reset authentication
+      def authenticate!
+        @access_expiry = nil
+        authenticate
+      end
+
+      def authenticate
+        if @access_expiry.to_i < Time.now.to_i
+          @access_code = nil
+          @auth.headers.delete :Authorization
+          @cap_api.headers.delete :Authorization
+        end
+        @access_code || begin
+          return false if @config.token_user.empty? && @config.token_pass.empty?
+          client = "#{@config.token_user}:#{@config.token_pass}"
+          auth_code = 'Basic ' + Base64.strict_encode64(client)
+          @auth.headers.merge!({ Authorization: auth_code })
+          response = @auth.get "?grant_type=client_credentials"
+          return false unless response.status == 200
+          access = response.body
+          return false if access['access_token'].nil?
+          @access_code = "Bearer #{access['access_token']}"
+          @access_expiry = Time.now.to_i + access['expires_in'].to_i
+          @cap_api.headers[:Authorization] = @access_code
+        end
+      end
+
+      # Get profiles from CAP API and store into local repo
+      def get_profiles
+        begin
+          if authenticate
+            page = 1
+            pages = 0
+            total = 0
+            begin
+              repo_clean
+              while true
+                params = "?p=#{page}&ps=100"
+                response = @cap_api.get "#{@cap_profiles}#{params}"
+                if response.status == 200
+                  data = response.body
+                  if data['firstPage']
+                    pages = data['totalPages']
+                    total = data['totalCount']
+                    puts "Retrieved #{page} of #{pages} pages (#{total} profiles)."
+                  else
+                    puts "Retrieved #{page} of #{pages} pages."
+                  end
+                  profiles = data['values']
+                  if @profiles.is_a? Daybreak::DB
+                    profiles.each do |profile|
+                      id = profile["profileId"]
+                      @profiles[id] = profile
+                    end
+                    @profiles.flush
+                  elsif @profiles.is_a? Mongo::Collection
+                    # split out the publication data to accommodate the
+                    # 16Mb limit on mongodb docs.
+                    pubs_fields = ['doiId', 'doiUrl', 'webOfScienceId', 'webOfScienceUrl']
+                    profiles.each do |profile|
+                      id = profile['profileId']
+                      presentations = profile.delete('presentations') || []
+                      presentations.each {|p| p.delete('detail')}
+                      pres = {'profileId' => id, 'presentations' => presentations}
+                      begin
+                        @presentations.insert_one(pres)
+                      rescue
+                        msg = "Profile #{id} presentations failed to save."
+                        @config.logger.error msg
+                      end
+                      publications = profile.delete('publications') || []
+                      publications.each do |p|
+                        p.keys {|k| p.delete(k) unless pubs_fields.include? k }
+                      end
+                      pub = {'profileId' => id, 'publications' => publications}
+                      begin
+                        @publications.insert_one(pub)
+                      rescue
+                        msg = "Profile #{id} publications failed to save."
+                        @config.logger.error msg
+                      end
+                      begin
+                        @profiles.insert_one(profile)
+                      rescue
+                        msg = "Profile #{id} failed to save."
+                        @config.logger.error msg
+                      end
+                    end
+                  end
+                  page += 1
+                  break if data['lastPage']
+                else
+                  msg = "Failed to GET profiles page #{page}: #{response.status}"
+                  @config.logger.error msg
+                  puts msg
+                  break
+                end
+              end
+            rescue => e
+              msg = e.message
+              binding.pry if @config.debug
+              @config.logger.error msg
+            ensure
+              repo_commit(total)
+            end
+          else
+            msg = "Failed to authenticate"
+            @config.logger.error msg
+          end
+        rescue => e
+          msg = e.message
+          binding.pry if @config.debug
+          @config.logger.error(msg)
+        end
+      end
+
+      # def update_profiles
+      #   # profile['profileId']
+      #   # => 42005
+      #   # [13] pry(main)> profile['profileId'].class
+      #   # => Fixnum
+      #   # [14] pry(main)> profile['lastModified']
+      #   # => "2015-08-17T10:55:46.772-07:00"
+      # end
+
+      # @return ids [Array<Integer>] profile ids from local repo
+      def profile_ids
+        if @profiles.is_a? Daybreak::DB
+          @profiles.keys.map {|k| k.to_i}
+        elsif @profiles.is_a? Mongo::Collection
+          @profiles.find.projection({profileId:1}).map {|p| p['profileId'] }
+        end
+      end
+
+      # return profile data from local repo
+      # @param id [Integer] A profileId number
+      # @return profile [Hash|nil]
+      def profile(id)
+        if @profiles.is_a? Daybreak::DB
+          @profiles[id.to_s]
+        elsif @profiles.is_a? Mongo::Collection
+          @profiles.find({profileId: id}).first
+        end
+      end
+
+      # return presentation data from local repo
+      # @param id [Integer] A profileId number
+      # @return presentations [Array<Hash>|nil]
+      def presentation(id)
+        if @profiles.is_a? Daybreak::DB
+          begin
+            @profiles[id.to_s]['presentations']
+          rescue
+            nil
+          end
+        elsif @profiles.is_a? Mongo::Collection
+          @presentations.find({profileId: id}).first
+        end
+      end
+
+      # return publication data from local repo
+      # @param id [Integer] A profileId number
+      def publication(id)
+        if @profiles.is_a? Daybreak::DB
+          begin
+            @profiles[id.to_s]['publications']
+          rescue
+            nil
+          end
+        elsif @profiles.is_a? Mongo::Collection
+          @publications.find({profileId: id}).first
+        end
+      end
+
+      # A profile's processing data.
+      # @param id [Integer] A profileId number
+      def processed(id)
+        if @profiles.is_a? Daybreak::DB
+          begin
+            @profiles[id.to_s]['processed']
+          rescue
+            nil
+          end
+        elsif @processed.is_a? Mongo::Collection
+          @processed.find({profileId: id}).first
+        end
+      end
+
+      # Update a profile record with processing data.
+      # @param id [Integer] A profileId number
+      # @param data [Hash] Optional processing information
+      def process_update(id, data=nil)
+        if @profiles.is_a? Daybreak::DB
+          process_doc = {
+            lastModified: Time.now.to_i,
+            data: data
+          }
+          @profiles[id.to_s]['processed'] = process_doc
+        elsif @processed.is_a? Mongo::Collection
+          process_doc = {
+            profileId: id,
+            lastModified: Time.now.to_i,
+            data: data
+          }
+          @processed.insert_one(process_doc)
+        end
+      end
+
+      private
+
+      def repo_clean
+        if @profiles.is_a? Daybreak::DB
+          @profiles.clear
+        elsif @profiles.is_a? Mongo::Collection
+          @profiles.drop
+          @profiles.create
+          @presentations.drop
+          @presentations.create
+          @publications.drop
+          @publications.create
+          @processed.drop
+          @processed.create
+        end
+      end
+
+      def repo_commit(total)
+        if @profiles.is_a? Daybreak::DB
+          @profiles.flush
+          @profiles.compact
+          @profiles.load
+          puts "Stored #{@profiles.size} of #{total} profiles."
+          puts "Stored profiles to #{@profiles.class} at: #{@profiles.file}."
+        elsif @profiles.is_a? Mongo::Collection
+          indexes = {"profileId" => 1}
+          @profiles.indexes.create_one( indexes, :unique => true )
+          @presentations.indexes.create_one( indexes, :unique => true )
+          @publications.indexes.create_one( indexes, :unique => true )
+          @processed.indexes.create_one( indexes, :unique => true )
+          puts "Stored #{@profiles.find.count} of #{total} profiles."
+          puts "Stored profiles to #{@profiles.class} at: #{@profiles.namespace}."
+        end
+      end
+
+      # Migrate CAP API profile data from a Daybreak::DB into mongodb
+      def profiles_daybreak_to_mongo
+        mongo = Cap.configuration.cap_repo_mongo
+        mongo[:profiles].drop
+        db = Cap.configuration.cap_repo_daybreak
+        db.keys do |id|
+          profile = profiles[id]
+          mongo[:profiles].insert_one(profile)
+        end
+        mongo[:profiles].indexes.create_one({profileId:1}, :unique => true )
+        daybreak_matches_mongo?
+      end
+
+      # Validate a daybreak to mongo data transfer
+      def daybreak_matches_mongo?
+        mongo = Cap.configuration.cap_repo_mongo
+        profiles = Cap.configuration.cap_repo_daybreak
+        matches = profiles.keys.map do |id|
+          profile = profiles[id]
+          mongo_profile = mongo[:profiles].find(:profileId => id.to_i).first
+          mongo_profile.delete("_id")
+          mongo_profile == profile
+        end
+        matches.all?  # should be true
+      end
+
+      def json_payloads
+        { accept: JSON_CONTENT, content_type: JSON_CONTENT }
+      end
+
+    end
+  end
+end