gared 0.0.22 → 0.0.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12ad7a1d34ae412a0828d5a9ea7297743d631781248f2ecbcfe646e316cfc42e
-  data.tar.gz: 4c2dab22f6984e50971164a9c984b6034053e0c86a717130afe07b7c743b5fcf
+  metadata.gz: cd1e7f535a0650e0251fec57b3572e49dda88b189b96fe1d1b630acc5e147bb4
+  data.tar.gz: b606f6169bdf2cac88f5d29cec6bf4e5464a639bc73b655a24c61aecf7593cd1
 SHA512:
-  metadata.gz: 3825a5a4bac92b0bb92f3b5fe5955c2a7780a489ceaae7718fc4eead41fa9788b567b2359a8a75efa51c88b8d65623580d27b93e3e29d95f41858ea9b76e66e9
-  data.tar.gz: 299626204c6f0b3f8cffdb46cc0548c41d7583fc094e3d0455ccead2f2198c4ec9805914036c8c5144874d6487ce980024ce52d016ce3955336351123204a9a2
+  metadata.gz: b5c082ab85a8a7fb4cf5c23af7ebc35584751ebcaf88defc1b4f8c8d808d5cb3ff5d213db4b07687bfae49fda11b12f6963c9c6d2bd2e6b1d2e5edcc487e6e4b
+  data.tar.gz: a4415463242669077a504252094c2e93f63469fe6fcd4faeeee7a5cceb852b5325608dd5255991c79f04d725856ac61296023301c39af0a4f48288dbe1ddad70

data/lib/gared.rb

@@ -7,6 +7,7 @@ module Gared
   require 'gared/hebrewbooks'
   require 'gared/idea'
   require 'gared/googlebooks'
+  require 'gared/nli_api'
   # ...
 
 end

data/lib/gared/jpress.rb

@@ -1,35 +1,4 @@
 module Gared
   class Jpress
-    require 'watir'
-
-    def initialize
-      @browser = Watir::Browser.new :chrome, options: {args: ['--no-sandbox', '--headless']}
-    end
-
-    def query_persons(q)
-    end
-
-    def query_person(person)
-    end
-
-    def query_publications(q)
-    end
-
-    def query_publication(publication)
-    end
-
-    # return in-memory Publication instances with associated Holdings
-    def query_publications_by_person(person, ctx = nil)
-      @browser.goto 'http://web.nli.org.il/sites/JPress/Hebrew/Pages/default.aspx'
-      @browser.wait
-
-      t = @browser.text_field(id: 'ctl00_PlaceHolderHeader_PlaceHolderSearchArea_ctl00_ctl00_ctl00_SD01C0892_InputKeywords') # srsly, Micro$oft
-      t.set(person)
-      @browser.a(id: 'ctl00_PlaceHolderHeader_PlaceHolderSearchArea_ctl00_ctl00_ctl00_SD01C0892_go').click # "quick search" - not necessarily by author!
-      @browser.wait
-      ret = []
-      begin
-
-    end
   end
 end

data/lib/gared/nli_api.rb

@@ -0,0 +1,89 @@
+require 'rest-client'
+
+module Gared
+  class Nli_Api
+    def initialize(url, api_key)
+      @options = {url: url, api_key: api_key}
+    end
+
+    def query_persons(q)
+    end
+
+    def query_person(person)
+    end
+
+    def query_publications(q)
+    end
+
+    def query_publication(publication)
+    end
+    def fetch_value_by_dc_key(record, key)
+      ret = ''
+      fullkey = key[0] == '@' ? key : 'http://purl.org/dc/elements/1.1/' + key
+      unless record.nil?
+        unless record[fullkey].nil?
+          if record[fullkey].class == String
+            ret = record[fullkey  ]
+          elsif record[fullkey].class == Array
+            ret = record[fullkey].map{|x| x['@value'] }.join('; ')
+          end
+        end
+      end
+      ret
+    end
+    # return in-memory Publication instances with associated Holdings
+    def query_publications_by_person(person, ctx = nil)
+      ret = []
+      begin
+        # first run obtain counts for the query
+        escaped_person = URI.escape(person)
+        url = @options[:url]+"?api_key=#{@options[:api_key]}&query=creator,contains,#{escaped_person},AND;language,exact,heb&sort_field=title&material_type=books&count_mode=true"
+        json = JSON.parse(RestClient::Resource.new(url,verify_ssl: OpenSSL::SSL::VERIFY_NONE).get)
+        total = json['countInfos']['total']
+        # then start loading the results
+        result_page = 1
+        recs = []
+        while recs.length < total
+          url = @options[:url]+"?api_key=#{@options[:api_key]}&query=creator,contains,#{escaped_person},AND;language,exact,heb&sort_field=title&material_type=books&result_page=#{result_page}"
+          puts "DBG: retrieving results page #{result_page}"
+          json = JSON.parse(RestClient::Resource.new(url,verify_ssl: OpenSSL::SSL::VERIFY_NONE).get)
+          recs += json
+          result_page += 1
+          # sleep 1 # respect the server and avoid flood-blocking
+        end
+        recs.each do |r|
+          begin
+            p = Publication.new(ctx)
+            p.title = fetch_value_by_dc_key(r, 'title')
+            p.author_line = fetch_value_by_dc_key(r, 'creator')
+            p.language = fetch_value_by_dc_key(r, 'language')
+            p.notes = "#{fetch_value_by_dc_key(r, 'format')}\n#{fetch_value_by_dc_key(r, 'subject')}"
+            p.publisher_line = fetch_value_by_dc_key(r,'publisher')
+            p.pub_year = fetch_value_by_dc_key(r, 'non_standard_date')
+            p.source_id = fetch_value_by_dc_key(r, '@id')
+            # collect additional URLS from record, for clients to be able to determine whether a scanned object exists
+            additional_urls = []
+            r.keys.each do |key|
+              val = fetch_value_by_dc_key(r, key)
+              additional_urls << val if val =~ /https?:[^\s]\/\//
+            end
+            p.additional_urls = additional_urls if additional_urls.length > 0
+            h = Holding.new
+            h.source_id = p.source_id
+            h.source_name = 'NLI API'
+            h.location = fetch_value_by_dc_key(r, 'recordid')
+            p.add_holding(h)
+            ret << p
+          rescue Exception
+            puts $!
+          end
+        end
+        # TODO: also collect IIIF links for the *subset* of titles that have them, using the availability_type param.  No way to get that in the above query -- the fields are not emitted.
+        # the URL is like https://api.nli.org.il/openlibrary/search?api_key=(((KEY)))&query=title,contains,querystring&availability_type=online_and_api_access&material_type=books
+      rescue Exception
+        puts $!
+      end
+      return ret
+    end  
+  end
+end

data/test/test_gared.rb

@@ -1,8 +1,24 @@
+require 'minitest/byebug' if ENV['DEBUG']
 require 'minitest/autorun'
 require 'gared'
 
 class GaredTest < Minitest::Test
 
+  def test_nli_api_query_publicatios_by_person
+    if ENV['NLI_API_KEY'].nil?
+      puts "skipping NLI API test because NLI_API_KEY envvar is not set"
+      return
+    end
+    puts "Testing NLI API"
+    nli = Gared::Nli_Api.new('https://api.nli.org.il/openlibrary/search', ENV['NLI_API_KEY'])
+    refute_nil nli
+    recs = nli.query_publications_by_person('ביאליק')
+    #recs = nli.query_publications_by_person('אילנאה')
+    refute_nil recs
+    refute_empty(recs)
+    refute_empty(recs[0].title)
+  end
+
   def test_primo_query_publicatios_by_person
     puts "Testing Primo"
     primo = Gared::Primo.new('http://primo.nli.org.il/PrimoWebServices/xservice/search/brief', 'NNL')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gared
 version: !ruby/object:Gem::Version
-  version: 0.0.22
+  version: 0.0.23
 platform: ruby
 authors:
 - Asaf Bartov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-19 00:00:00.000000000 Z
+date: 2021-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zoom
@@ -93,6 +93,7 @@ files:
 - lib/gared/holding.rb
 - lib/gared/idea.rb
 - lib/gared/jpress.rb
+- lib/gared/nli_api.rb
 - lib/gared/person.rb
 - lib/gared/primo.rb
 - lib/gared/publication.rb
@@ -116,8 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Scrape Hebrew bibliography sources