gared 0.0.21 → 0.0.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b31908b108520003b9d2f7e4c37a3d7f8ccab124f577343b5e5a7b7f1473432
-  data.tar.gz: 7ebd7bf7246c5c703f021f403460aa8b3ba1107842df23ebc1e6b0f40e89b9a5
+  metadata.gz: 3857f28b69b7cd0d1080208215cbf1bfc57a30a3af5df8162a98aee5b7558939
+  data.tar.gz: a3cf8fbe5018b902db45f6d70c8223f32101c6eb747b070a7f7dc56a8a37c48c
 SHA512:
-  metadata.gz: 169547fe7caaf6f0777460b7c76cae17c6583a4b3c064e130dd6ddc33bf50bf7984576b216a3c8554dd08f6911a7b2f930e6b4d6f1b2713df729c2c3fc3617ed
-  data.tar.gz: dea903b2aee797c53f62956645bdfa99737fa29eec4019f670a139cac34ee46dca97954b52b217594c0d314d9513b2f6f08f9cea8b474b94248101d18d3628f1
+  metadata.gz: 69f3b46c6fa96f5bf2c2f440fbc9bf93bbe57ec8ac61e84a9621289e67b919342558a323a9b14008d58f7472e48651ee43bd584879b011d78741a6212b7e81b1
+  data.tar.gz: f822f3a7f95e6c4ca38f78843a4309b764a624d053234bc880b672ec57cf011b88b3089f5a40836f328a8e1a366cb6660500609449a11eed16d5b0c7a86ac64e

data/lib/gared/googlebooks.rb

@@ -5,9 +5,13 @@ module Gared
     def initialize(api_key, page_size = '40')
       @options = {api_key: api_key, maxResults: page_size}
     end
-
+    def uri_escape(s)
+      p = URI::Parser.new
+      return p.escape(s)
+    end
+  
     def query_publications_by_person(person, ctx = nil)
-      url = "https://www.googleapis.com/books/v1/volumes?q=inauthor:#{URI.escape(person)}&filter=full&key=#{@options[:api_key]}&maxResults=#{@options[:maxResults]}"
+      url = "https://www.googleapis.com/books/v1/volumes?q=inauthor:#{uri_escape(person)}&filter=full&key=#{@options[:api_key]}&maxResults=#{@options[:maxResults]}"
       resp = JSON.parse(RestClient.get(url))
     
       total = resp['totalItems']

data/lib/gared/jpress.rb

@@ -1,35 +1,4 @@
 module Gared
   class Jpress
-    require 'watir'
-
-    def initialize
-      @browser = Watir::Browser.new :chrome, options: {args: ['--no-sandbox', '--headless']}
-    end
-
-    def query_persons(q)
-    end
-
-    def query_person(person)
-    end
-
-    def query_publications(q)
-    end
-
-    def query_publication(publication)
-    end
-
-    # return in-memory Publication instances with associated Holdings
-    def query_publications_by_person(person, ctx = nil)
-      @browser.goto 'http://web.nli.org.il/sites/JPress/Hebrew/Pages/default.aspx'
-      @browser.wait
-
-      t = @browser.text_field(id: 'ctl00_PlaceHolderHeader_PlaceHolderSearchArea_ctl00_ctl00_ctl00_SD01C0892_InputKeywords') # srsly, Micro$oft
-      t.set(person)
-      @browser.a(id: 'ctl00_PlaceHolderHeader_PlaceHolderSearchArea_ctl00_ctl00_ctl00_SD01C0892_go').click # "quick search" - not necessarily by author!
-      @browser.wait
-      ret = []
-      begin
-
-    end
   end
 end

data/lib/gared/nli_api.rb

@@ -0,0 +1,93 @@
+require 'rest-client'
+
+module Gared
+  class Nli_Api
+    def initialize(url, api_key)
+      @options = {url: url, api_key: api_key}
+    end
+    def uri_escape(s)
+      p = URI::Parser.new
+      return p.escape(s)
+    end
+  
+    def query_persons(q)
+    end
+
+    def query_person(person)
+    end
+
+    def query_publications(q)
+    end
+
+    def query_publication(publication)
+    end
+    def fetch_value_by_dc_key(record, key)
+      ret = ''
+      fullkey = key[0] == '@' ? key : 'http://purl.org/dc/elements/1.1/' + key
+      unless record.nil?
+        unless record[fullkey].nil?
+          if record[fullkey].class == String
+            ret = record[fullkey  ]
+          elsif record[fullkey].class == Array
+            ret = record[fullkey].map{|x| x['@value'] }.join('; ')
+          end
+        end
+      end
+      ret
+    end
+    # return in-memory Publication instances with associated Holdings
+    def query_publications_by_person(person, ctx = nil)
+      ret = []
+      begin
+        # first run obtain counts for the query
+        escaped_person = uri_escape(person)
+        url = @options[:url]+"?api_key=#{@options[:api_key]}&query=creator,contains,#{escaped_person},AND;language,exact,heb&sort_field=title&material_type=book&count_mode=true"
+        json = JSON.parse(RestClient::Resource.new(url,verify_ssl: OpenSSL::SSL::VERIFY_NONE).get)
+        total = json['countInfos']['total']
+        # then start loading the results
+        result_page = 1
+        recs = []
+        while recs.length < total
+          url = @options[:url]+"?api_key=#{@options[:api_key]}&query=creator,contains,#{escaped_person},AND;language,exact,heb&sort_field=title&material_type=book&result_page=#{result_page}"
+          puts "DBG: retrieving results page #{result_page}"
+          json = JSON.parse(RestClient::Resource.new(url,verify_ssl: OpenSSL::SSL::VERIFY_NONE).get)
+          recs += json
+          result_page += 1
+          # sleep 1 # respect the server and avoid flood-blocking
+        end
+        recs.each do |r|
+          begin
+            p = Publication.new(ctx)
+            p.title = fetch_value_by_dc_key(r, 'title')
+            p.author_line = fetch_value_by_dc_key(r, 'creator')
+            p.language = fetch_value_by_dc_key(r, 'language')
+            p.notes = "#{fetch_value_by_dc_key(r, 'format')}\n#{fetch_value_by_dc_key(r, 'subject')}"
+            p.publisher_line = fetch_value_by_dc_key(r,'publisher')
+            p.pub_year = fetch_value_by_dc_key(r, 'non_standard_date')
+            p.source_id = fetch_value_by_dc_key(r, '@id')
+            # collect additional URLS from record, for clients to be able to determine whether a scanned object exists
+            additional_urls = []
+            r.keys.each do |key|
+              val = fetch_value_by_dc_key(r, key)
+              additional_urls << val if val =~ /https?:[^\s]\/\//
+            end
+            p.additional_urls = additional_urls if additional_urls.length > 0
+            h = Holding.new
+            h.source_id = p.source_id
+            h.source_name = 'NLI API'
+            h.location = fetch_value_by_dc_key(r, 'recordid')
+            p.add_holding(h)
+            ret << p
+          rescue Exception
+            puts $!
+          end
+        end
+        # TODO: also collect IIIF links for the *subset* of titles that have them, using the availability_type param.  No way to get that in the above query -- the fields are not emitted.
+        # the URL is like https://api.nli.org.il/openlibrary/search?api_key=(((KEY)))&query=title,contains,querystring&availability_type=online_and_api_access&material_type=books
+      rescue Exception
+        puts $!
+      end
+      return ret
+    end  
+  end
+end

data/lib/gared/primo.rb

@@ -6,7 +6,11 @@ module Gared
     def initialize(url, institution)
       @options = {url: url, institution: institution}
     end
-
+    def uri_escape(s)
+      p = URI::Parser.new
+      return p.escape(s)
+    end
+  
     def query_persons(q)
     end
 
@@ -23,14 +27,14 @@ module Gared
     def query_publications_by_person(person, ctx = nil)
       ret = []
       begin
-        url = @options[:url]+"?institution=#{@options[:institution]}&query=creator,contains,#{URI.escape(person)}&indx=1&bulkSize=50&query=facet_rtype,exact,books&json=true"
+        url = @options[:url]+"?institution=#{@options[:institution]}&query=creator,contains,#{uri_escape(person)}&indx=1&bulkSize=50&query=facet_rtype,exact,books&json=true"
         json = JSON.parse(RestClient.get(url))
         total = json['SEGMENTS']['JAGROOT']['RESULT']['DOCSET']['@TOTALHITS'].to_i
         start_at = 1
         recs = json['SEGMENTS']['JAGROOT']['RESULT']['DOCSET']['DOC'] # stash the records
         while recs.length < total
           start_at += 50
-          url = @options[:url]+"?institution=#{@options[:institution]}&query=creator,contains,#{URI.escape(person)}&indx=#{start_at}&bulkSize=50&query=facet_rtype,exact,books&json=true"
+          url = @options[:url]+"?institution=#{@options[:institution]}&query=creator,contains,#{uri_escape(person)}&indx=#{start_at}&bulkSize=50&query=facet_rtype,exact,books&json=true"
           json = JSON.parse(RestClient.get(url))
           recs += json['SEGMENTS']['JAGROOT']['RESULT']['DOCSET']['DOC']
           sleep 1 # respect the server and avoid flood-blocking
@@ -55,7 +59,8 @@ module Gared
             h = Holding.new
             h.source_id = p.source_id
             h.source_name = 'Primo:'+@options[:institution]
-            h.location = r['LIBRARIES']['LIBRARY'][0]['callNumber']
+            
+            h.location = r['LIBRARIES']['LIBRARY'][0].nil? ? r['LIBRARIES']['LIBRARY']['callNumber'] : r['LIBRARIES']['LIBRARY'][0]['callNumber'] # there seem to be two cases, different between NLI and TAU, for example
             p.add_holding(h)
             ret << p
           rescue Exception

data/lib/gared.rb

@@ -7,6 +7,6 @@ module Gared
   require 'gared/hebrewbooks'
   require 'gared/idea'
   require 'gared/googlebooks'
+  require 'gared/nli_api'
   # ...
-
 end

data/test/test_gared.rb

@@ -1,31 +1,53 @@
+require 'minitest/byebug' if ENV['DEBUG']
 require 'minitest/autorun'
 require 'gared'
 
 class GaredTest < Minitest::Test
 
-  def test_primo_query_publicatios_by_person
-    puts "Testing Primo"
-    primo = Gared::Primo.new('http://primo.nli.org.il/PrimoWebServices/xservice/search/brief', 'NNL')
-    refute_nil primo
-    recs = primo.query_publications_by_person('אילנאה')
+  def test_nli_api_query_publicatios_by_person
+    if ENV['NLI_API_KEY'].nil?
+      puts "skipping NLI API test because NLI_API_KEY envvar is not set"
+      return
+    end
+    puts "Testing NLI API"
+    nli = Gared::Nli_Api.new('https://api.nli.org.il/openlibrary/search', ENV['NLI_API_KEY'])
+    refute_nil nli
+    #recs = nli.query_publications_by_person('ביאליק')
+    recs = nli.query_publications_by_person('אילנאה')
     refute_nil recs
     refute_empty(recs)
     refute_empty(recs[0].title)
   end
 
-  def test_aleph_query_publicatios_by_person
-    puts "Testing Aleph"
-    aleph = Gared::Aleph.new('aleph.nli.org.il', 9991, 'NNL01')
-    refute_nil aleph
-    recs = aleph.query_publications_by_person('אילנאה')
-    refute_nil recs
-    refute_empty(recs)
-    refute_empty(recs[0].title)
-  end
+# temporarily disabled until we find another Primo server to test against
+#  def test_primo_query_publicatios_by_person
+#    puts "Testing Primo"
+#    primo = Gared::Primo.new('http://primo.nli.org.il/PrimoWebServices/xservice/search/brief', 'NNL')
+#    refute_nil primo
+#    recs = primo.query_publications_by_person('אילנאה')
+#    refute_nil recs
+#    refute_empty(recs)
+#    refute_empty(recs[0].title)
+#  end
+
+# temporarily disabled until we find another Aleph server to test against
+#  def test_aleph_query_publicatios_by_person
+#    puts "Testing Aleph"
+#    aleph = Gared::Aleph.new('aleph.nli.org.il', 9991, 'NNL01')
+#    refute_nil aleph
+#    recs = aleph.query_publications_by_person('אילנאה')
+#    refute_nil recs
+#    refute_empty(recs)
+#    refute_empty(recs[0].title)
+#  end
 
   def test_googlebooks_query_publicatios_by_person
+    if ENV['GOOGLE_API_KEY'].nil?
+      puts "skipping Google Books API test because GOOGLE_API_KEY envvar is not set"
+      return
+    end
     puts "Testing Google Books"
-    gb = Gared::Googlebooks.new('AIzaSyCE2WFqTPdxAz1wv2f33hMfPWIF4tcocgM') # a key I made just for testing this gem. Please do not abuse.
+    gb = Gared::Googlebooks.new(ENV['GOOGLE_API_KEY'])
     refute_nil gb
     recs = gb.query_publications_by_person('מנדלי')
     refute_nil recs
@@ -58,4 +80,4 @@ class GaredTest < Minitest::Test
     assert_empty(recs)
   end
 
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gared
 version: !ruby/object:Gem::Version
-  version: 0.0.21
+  version: 0.0.26
 platform: ruby
 authors:
 - Asaf Bartov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-19 00:00:00.000000000 Z
+date: 2022-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zoom
@@ -93,6 +93,7 @@ files:
 - lib/gared/holding.rb
 - lib/gared/idea.rb
 - lib/gared/jpress.rb
+- lib/gared/nli_api.rb
 - lib/gared/person.rb
 - lib/gared/primo.rb
 - lib/gared/publication.rb
@@ -116,8 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Scrape Hebrew bibliography sources