libis-metadata 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 23ff1ec087acd8448cd8230000f6d5b3aa5f7acf
+  data.tar.gz: a908b2af8489af83de9284e529abdfd4e7b68f08
+SHA512:
+  metadata.gz: aa677a22babb3b23d1066901db39d6b0d0a40832bdc6d8a28aa14cd0ff21e616ca4ad4c67adf990c7a5a4dcb00068b9c3345e072fce28e42eb7335db57ed8ace
+  data.tar.gz: 3223dace5cc94e68359694f2dfaf50640b55af25a156b083bc0ca793abb403285b5fd77df4eee8553d241207be624504a22562da42bcd71204567c153ee2b553

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5.0
+before_install: gem install bundler -v 1.16.1

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in metadata.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Kris Dekeyser
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# Metadata
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/metadata`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'libis-metadata'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install libis-metadata
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/metadata.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "libis-metadata"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/metadata

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'libis/metadata'
+
+Libis::Metadata::CommandLine.start(ARGV)

data/lib/libis/metadata/cli/cli_downloader.rb

@@ -0,0 +1,182 @@
+require 'libis/tools/spreadsheet'
+require 'awesome_print'
+
+module Libis
+  module Metadata
+    module Cli
+      module Downloader
+
+        # noinspection RubyStringKeysInHashInspection
+        VALID_SOURCES = {
+            'alma' => 'alma.mms_id',
+            'scope' => 'ID'
+        }
+
+        REQ_HEADERS = {term: 'Term'}
+        OPT_HEADERS = {pid: 'Pid', filename: 'File'}
+
+        def self.included(klass)
+          klass.class_exec do
+            desc 'download [options] [TERM[:PID] ...]', 'Download metadata from Alma or Scope'
+            long_desc <<-DESC
+      
+            'download [TERM ...]' will download metadata from Alma or Scope and convert it to Dublin Core.
+          
+            The output format can either be Dublin Core or a Rosetta MD Update file. If you supply a Rosetta IE PID the 
+            tool will generate a Rosetta MD Update file, a Dublin Core XML file if you don't. Note that there is no 
+            check if the IE PID is a valid one.
+          
+            Any TERM argument that starts with a '@' will be interpreted as an input file name. The input file name can
+            be:
+
+            * a simple text file. File extension should be '.txt' and each non-empty line is interpreted as if it was
+              supplied as a TERM argument on the command line.
+
+            * a comma-delimited file (CSV). File extension should be '.csv'.
+
+            * a tab-deliimited file (TSV). File extension should be '.tsv'.
+
+            * a spreadsheet. Excel files (.xls or xlsx) and OpenOffice/LibreOffice Calc files (.ods) are supported.
+              '@<sheet_name>' must be appended to the file name to select the proper sheet tab.
+
+            For the CSV, TSV and spreadsheets: if there is no header row, the first column should contain the search 
+            terms. If present, the second column should contain PID info and the third column FILE info. Other columns 
+            are ignored. If there is a header row, it should contain at least a cell with the text 'Term'. That column 
+            is expexted to contain the search terms. If a column header with the text 'Pid' is found, the column data 
+            will be expected to contain pids for the IE's to modify. If a column header with the text 'File' is found, 
+            the column data will be expected to contain file names to save to.
+
+            In any case, if the output file info is missing, the name defaults to the PID (if present) or the search 
+            term. If the FILE info does not have a file extension, '.xml' will be added. 
+            
+            The list of TERM arguments will be processed automatically. If there are no terms supplied on the command 
+            line, the program will ask for them until you supply an empty value. A TERM argument can contain PID and 
+            FILE info separated by a ':' or whatever you supply for the separator option. TERM arguments supplied via 
+            a simple text file are interpreted the same way.
+
+            Examples:
+            * abc => searches for 'abc' and save DC metadata in abc.xml
+            * abc:123 => searches for 'abc' and generates MD Update in 123.xml
+            * abc:123:xyz.data => searches for 'abc' and generates MD Update in xyz.data
+            * abc::xyz => searches for 'abc' and save DC metadata in xyz.xml
+
+            For any option that is not supplied on the command line and doesn't have a default value, the tool will 
+            always ask you to supply a value, even if the '-q' option is given.
+
+            DESC
+
+            method_option :quiet, aliases: '-q', desc: 'Do not ask for options that have a default value',
+                          type: :boolean, default: false
+            method_option :metadata, aliases: '-m', banner: 'source', desc: 'Metadata source system',
+                          default: VALID_SOURCES.keys.first, enum: VALID_SOURCES.keys
+            method_option :field, aliases: '-f', banner: 'field_name', desc: 'Search field in the Metadata system;' +
+                " default value depends on selected metadata source system: #{VALID_SOURCES}"
+            method_option :library, aliases: '-l', banner: 'library_code', desc: 'Library code for Alma',
+                          default: '32KUL_KUL'
+            method_option :database, aliases: '-d', desc: 'Scope database to connect to'
+            method_option :user, aliases: '-u', desc: 'Database user name'
+            method_option :password, aliases: '-p', desc: 'Database password'
+            method_option :target_dir, aliases: '-t', desc: 'Directory where files will be created', default: '.'
+            method_option :separator, aliases: '-s', desc: 'Separator for the TERM arguments', default: ':'
+
+          end
+        end
+
+        def download(*terms)
+
+          unless Dir.exist?(options.target_dir)
+            prompt.error "ERROR: target directory '#{options.target_dir}' does not exist"
+            exit -1
+          end
+
+          md = Libis::Metadata::Downloader.new
+
+          config = download_configure
+
+          md.configure config
+
+          if terms.empty?
+            while (term = prompt.ask "Search #{config[:metadata]} for #{config[:field]}:")
+              pid = prompt.ask 'IE PID to update:'
+              filename = prompt.ask('File name:', default: "#{pid || term}.xml")
+              download_one(service: md, term: term, pid: pid, filename: filename)
+            end
+          else
+            terms.each do |term|
+              if term =~ /^@((.+)@(.+)|(.+))$/
+                sheet = $3
+                file = $2 || $4
+                unless File.exist? file
+                  prompt.warn "WARNING: File name '#{file}' not found."
+                  next
+                end
+                if File.extname(file) == '.txt'
+                  File.open(file, 'r').each_line do |line|
+                    line.strip!
+                    next if line.empty?
+                    x = split_term(line)
+                    download_one(service: md, term: x[:term], pid: x[:pid], filename: x[:filename])
+                  end
+                else
+                  opts = {required: REQ_HEADERS, optional: OPT_HEADERS, noheaders: REQ_HEADERS.merge(OPT_HEADERS)}
+                  opts.merge!(col_sep: "\t", extension: :csv) if File.extname(file) == '.tsv'
+                  Libis::Tools::Spreadsheet.foreach("#{file}#{sheet ? '|' + sheet : ''}", opts) do |row|
+                    next if row[:term].nil? || row[:term] == 'Term'
+                    ap row
+                    download_one(service: md, term: row[:term], pid: row[:pid], filename: row[:filename])
+                  end
+                end
+                next
+              end
+              x = split_term(term)
+              download_one(service: md, term: x[:term], pid: x[:pid], filename: x[:filename])
+            end
+          end
+        end
+
+        protected
+
+        def split_term(term)
+          result = {}
+          t, p, f = term.split(options.separator)
+          result[:term] = t
+          result[:pid] = p if p && !p.empty?
+          result[:filename] = f if f && !f.empty?
+          result
+        end
+
+        def download_one(service:, term:, pid: nil, filename: nil)
+          filename += '.xml' unless (filename.nil? || !File.extname(filename).empty?)
+          filename ||= "#{pid || term}.xml"
+          service.download(term, filename, pid)
+          prompt.ok "OK: #{filename}#{" [#{term}]"}"
+        rescue Exception => e
+          prompt.error e.message.strip
+        end
+
+        def download_configure
+          result = options.quiet? ? options.key_strings_to_symbols : {}
+
+          ask 'Metadata source:', result, :metadata, default: options.metadata, enum: VALID_SOURCES.keys
+          # Set appropriate default for field depending on source selection
+          result[:field] ||= VALID_SOURCES[result[:metadata]] if options.quiet?
+          ask 'Search field:', result, :field, default: options.field || VALID_SOURCES[result[:metadata]]
+          case result[:metadata]
+          when 'alma'
+            ask 'Library code:', result, :library, default: options.library
+          when 'scope'
+            ask 'Database name:', result, :database, default: options.database
+            ask 'User name:', result, :user, default: options.user
+            ask 'Password:', result, :password, default: options.password, mask: true
+          else
+            # Other source
+          end
+          result[:target_dir] = tree_select(options.target_dir, question: 'Target directory:') unless result[:target_dir]
+
+          result
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/metadata/cli/cli_helper.rb

@@ -0,0 +1,74 @@
+module Libis
+  module Metadata
+    module Cli
+      module Helper
+
+        module ClassMethods
+
+          def exit_on_failure?
+            true
+          end
+
+        end
+
+        def self.included(base)
+          base.extend(ClassMethods)
+        end
+
+        attr_reader :prompt
+
+        def initialize(*args)
+          @prompt = TTY::Prompt.new
+          super
+        end
+
+        private
+
+        def index_of(list, value)
+          i = list.index(value)
+          i += 1 if i
+          i
+        end
+
+        def ask(question, config, field, enum: nil, default: nil, mask: false)
+          cmd, args, opts = :ask, [question], {}
+          if enum
+            cmd = :select
+            args << enum
+            # Change default to its index in the enum
+            default = index_of(enum, default)
+            # Force the question if the supplied value is not valid
+            config[field] = nil unless enum.include? config[field]
+          end
+          cmd = :mask if mask
+          opts[:default] = default if default
+          config[field] = prompt.send cmd, *args, opts if config[field].nil?
+        end
+
+        def tree_select(path, question: nil, file: false, page_size: 22, filter: true, cycle: false)
+          path = Pathname.new(path) unless path.is_a? Pathname
+          path = path.realpath
+
+          dirs = path.children.select {|x| x.directory?}.sort
+          files = file ? path.children.select {|x| x.file?}.sort : []
+
+          choices = []
+          choices << {name: "#{path}", value: path, disabled: file ? '' : false}
+          choices << {name: '[..]', value: path.parent}
+
+          dirs.each {|d| choices << {name: "[#{d.basename}]", value: d}}
+          files.each {|f| choices << {name: f.basename.to_path, value: f}}
+
+          question ||= "Select #{'file or ' if files}directory"
+          selection = prompt.select question, choices,
+                                    per_page: page_size, filter: filter, cycle: cycle, default: file ? 2 : 1
+
+          return selection.to_path if selection == path || selection.file?
+
+          tree_select selection, file: file, page_size: page_size, filter: filter, cycle: cycle
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/metadata/command_line.rb

@@ -0,0 +1,25 @@
+require 'thor'
+require 'tty-prompt'
+
+require 'libis/metadata/downloader'
+require 'libis/tools/extend/hash'
+
+require 'libis/metadata/cli/cli_helper'
+require 'libis/metadata/cli/cli_downloader'
+
+module Libis
+  module Metadata
+
+    class CommandLine < Thor
+
+      include Cli::Helper
+      include Cli::Downloader
+
+      def download(*terms)
+        super
+      end
+
+    end
+
+  end
+end

data/lib/libis/metadata/downloader.rb

@@ -0,0 +1,117 @@
+require 'uri'
+require 'pathname'
+require 'awesome_print'
+
+require 'libis/services/alma/sru_service'
+require 'libis/services/scope/search'
+
+require 'libis/tools/metadata/marc21_record'
+require 'libis/tools/metadata/dublin_core_record'
+require 'libis/tools/xml_document'
+require 'libis/tools/extend/string'
+
+require 'libis/tools/metadata/mappers/kuleuven'
+require 'libis/tools/metadata/mappers/scope'
+
+module Libis
+  module Metadata
+
+    class Downloader
+      attr_reader :service, :mapper_class, :config
+
+      def initialize
+        @service = nil
+        @target_dir = nil
+        @config = nil
+        @mapper_class = nil
+      end
+
+      def configure(config)
+        metadata = config[:metadata]
+        case metadata
+        when 'alma'
+          @service ||= Libis::Services::Alma::SruService.new
+          @mapper_class = Libis::Tools::Metadata::Mappers::Kuleuven
+        when 'scope'
+          @service = ::Libis::Services::Scope::Search.new
+          @mapper_class = Libis::Tools::Metadata::Mappers::Scope
+          @service.connect(config[:password], config[:password], config[:database])
+        else
+          raise RuntimeError, "Service '#{service}' unknown"
+        end
+        @target_dir = config[:target_dir]
+        @config = config
+      rescue Exception => e
+        raise RuntimeError "failed to configure metadata service: #{e.message} @ #{e.backtrace.first}"
+      end
+
+      def download(term, filename, pid = nil)
+        record = search(term)
+        return nil unless record
+
+        record = md_update_xml(pid, record) if pid
+
+        record.save File.join(@target_dir, filename)
+
+        filename
+      end
+
+      # @return [Libis::Tools::Metadata::DublinCoreRecord]
+      def search(term)
+        record = case service
+                 when ::Libis::Services::Alma::SruService
+                   result = service.search(config[:field], URI::encode("\"#{term}\""), config[:library])
+                   raise RuntimeError "Multiple records found for #{config[:field]}=#{term}" if result.size > 1
+                   result.empty? ? nil : ::Libis::Tools::Metadata::Marc21Record.new(result.first.root)
+
+                 when ::Libis::Services::Scope::Search
+                   service.query(term, type: config[:field])
+                   service.next_record do |doc|
+                     ::Libis::Tools::Metadata::DublinCoreRecord.new(doc.to_xml)
+                   end
+
+                 else
+                   raise RuntimeError "Service '#{service}' unknown"
+
+                 end
+
+        unless record
+          raise RuntimeError, "No record found for #{config[:field]} = '#{term}'"
+        end
+
+        record.extend mapper_class
+        record.to_dc
+
+      rescue Exception => e
+        raise RuntimeError, "Search request failed: #{e.message}"
+      end
+
+      def save(record, filename)
+        return false unless record
+
+        record.save File.join(@target_dir, filename)
+
+        true
+      end
+
+      NO_DECL = Nokogiri::XML::Node::SaveOptions::FORMAT + Nokogiri::XML::Node::SaveOptions::NO_DECLARATION
+
+      def md_update_xml(pid, record)
+        Libis::Tools::XmlDocument.parse <<EO_XML
+<updateMD xmlns="http://com/exlibris/digitool/repository/api/xmlbeans">
+  <PID>#{pid}</PID>
+  <metadata>
+    <type>descriptive</type>
+    <subType>dc</subType>
+    <content>
+      <![CDATA[#{record.document.to_xml(save_with: NO_DECL)}]]>
+    </content>
+  </metadata>
+</updateMD>
+EO_XML
+      end
+
+    end
+
+  end
+end