oai_schedules 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e87f0fab9a90cc32bb6c3892a9ad72e6dae77c8b7efbc4ab3198c39b7d2a660
-  data.tar.gz: b59ddf59dd77df9dc1365a279dbe7ac3a9fb7b8d5f1bb7a33a8951f3d4ce1f48
+  metadata.gz: 3f254ace930dec7e151a5af485130dd23623548d6f9d0860a764b90ce217c4bb
+  data.tar.gz: bfe93c7f19a894976c53f3d2e3d720e048c77c116b92283034ba7b17d4efe386
 SHA512:
-  metadata.gz: d384485639a44d848f35eb3bc0a25a44fe2357cd84aeb37530a683a153aaba6f4b9bb24b7a316caf424e8f0c57b77b775b45f168c06339b3a6b6e35b1b2ee8d9
-  data.tar.gz: 84117dbd3b7f6757290fdee7a9f3388623330d8d293961946b6ee2058b2be8a7898b60822598bd8e8ac6bf2340d3cbabcba05124d1aa087e876b2a83192ea948
+  metadata.gz: bbd9df7881c619f2660dd6b622adf2fd7ec6fa35cde400ff3490b1001b3635c7d89605882e53c9b242cc992de6d0153a7ec5d25d5b41e0672853b997924900a0
+  data.tar.gz: 918414787e68f89b8dab0e692096e708864ca7b32a81d1341c0cc09e72fbb70d2cd886a137f024e5946b6176b7d4b1b20d606ec12d33c81894c65588ca9c712e

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [0.4.0] - 2025-03-28
+
+- Added logger to digestion function
+- Invalidated resumption token when either schedule set or format change
+- Extended error message on OAI URL fetch fail
+
 ## [0.3.0] - 2025-03-26
 
 - Added schedule name and content to digestion function

data/README.md

@@ -22,8 +22,13 @@ gem install oai_schedules
 ```ruby
 require 'oai_schedules/manager'
 
-f_show = lambda do |name, content, records, done|
-  # ... do your stuff with records ...
+f_show = lambda do |name, content, records, done, error, state, logger|
+  if error.nil?
+    # ... do your stuff with records ...
+  else
+    puts error.message
+  end
+  puts state
   if done
     puts "done full harvesting"
   end
@@ -58,8 +63,9 @@ At every iteration, every 2 seconds, it will get the *partial* records by using
 The code will do this by querying `https://eudml.org/oai/OAIHandler?verb=ListRecords&...` and adding the necessary
 query parameters.
 The custom function provided as `f_digest` will then be called at each iteration.
-This will be provided schedule `name` and `content`, the partial list of `records` as a hash,
-and a `done` flag (full harvesting complete).
+This will be provided schedule `name` and `content`, the partial list of `records` as a hash, 
+a `done` flag (full harvesting complete), an `error` exception if happened, the harvesting `state`,
+and the same logger used internally by the schedules manager.
 and it will write the new one to the state file, until no token is provided (end of the harvesting).
 As soon as the schedule is added, it is executed.
 It is possible to add all schedules in advance, then call `sleep` for infinite event loop.

data/examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json

@@ -0,0 +1,63 @@
+{
+  "interval": "P1W",
+  "repository": {
+    "uri": "https://eudml.org/oai/OAIHandler",
+    "repository_name": "REPOX Repository",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "mailto:gilberto.pedrosa@ist.utl.pt"
+    ],
+    "earliest_datestamp": "1970-01-01T00:00:00+00:00",
+    "deleted_records": "persistent",
+    "granularity": "YYYY-MM-DD",
+    "metadata_format": [
+      "oai_dc",
+      "ese",
+      "eudml-article2",
+      "eudml-book2"
+    ],
+    "set": [
+      "BDIM",
+      "BulDML",
+      "CEDRAM",
+      "DMLE",
+      "DML_CZ_Monograph",
+      "DML_CZ_Proceeding",
+      "DML_CZ_Serial",
+      "EDPS",
+      "ELibM",
+      "GALLICA",
+      "GDZ_Band",
+      "GDZ_Mathematica",
+      "GDZ_Monographs",
+      "GDZ_RusDML",
+      "HDML_Books",
+      "HDML_Conferences",
+      "HDML_Journals",
+      "MISANU",
+      "NUMDAM",
+      "NUMDAM_book",
+      "PLDML",
+      "PLDML_book",
+      "PMath"
+    ],
+    "id": "02df523af427deb93b7cb4600ca347f9297d0e31d51c2783c634459dac457bd0"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "b1671aad-e825-4b5a-b50b-d8591b425e2a",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "7889ce03-28d9-479d-bb9a-b239f179453a"
+  },
+  "format": "oai_dc",
+  "set": "CEDRAM",
+  "from": "1970-01-01T00:00:00+00:00",
+  "until": "1999-01-12T09:43:02+00:00",
+  "id": "dc34623d-2ae2-4e90-91d8-26f4ba29a056"
+}

data/examples/example_01.rb

@@ -4,8 +4,13 @@ require 'oai_schedules/manager'
 
 # usage with folder listener
 
-f_show = lambda do |name, content, records, done|
-  # ... do your stuff with records ...
+f_show = lambda do |name, content, records, done, error, state, logger|
+  if error.nil?
+    # ... do your stuff with records ...
+  else
+    puts error.message
+  end
+  puts state
   if done
     puts "done full harvesting"
   end

data/examples/example_02.rb

@@ -4,8 +4,13 @@ require 'oai_schedules/manager'
 
 # usage with programmatic schedules addition / modify / remove
 
-f_show = lambda do |name, content, records, done|
-  # ... do your stuff with records ...
+f_show = lambda do |name, content, records, done, error, state, logger|
+  if error.nil?
+    # ... do your stuff with records ...
+  else
+    puts error.message
+  end
+  puts state
   if done
     puts "done full harvesting"
   end

data/lib/oai_schedules/manager.rb

@@ -213,6 +213,7 @@ module OAISchedules
 
 
     def modify_schedule(name, content)
+      handle_schedule_state_at_schedule_change(name, content)
       @schedules[name][:content] = deep_copy(content)
       handle_schedule_task(name)
     end
@@ -222,6 +223,19 @@ module OAISchedules
 
     private
 
+
+    def handle_schedule_state_at_schedule_change(name, content)
+      # invalidate resumption token if either schedule format or set changes
+      state = @schedules[name][:state]
+      if    (content["format"] != @schedules[name][:content]["format"]) \
+        ||  (content["set"] != @schedules[name][:content]["set"])
+        state["resumption_token"] = nil
+        state["count_success"] = 0
+        state["count_fails"] = 0
+      end
+    end
+
+
     def deep_copy(o)
       Marshal.load(Marshal.dump(o))
     end
@@ -326,8 +340,11 @@ module OAISchedules
         when StateHarvesting::IDLE
           state_machine.add_event(EventHarvesting::REQUEST_HARVEST)
         when StateHarvesting::HARVESTING
-          unless state.has_key?("count_partial_harversting")
-            state["count_partial_harversting"] = 0
+          unless state.has_key?("count_success")
+            state["count_success"] = 0
+          end
+          unless state.has_key?("count_fails")
+            state["count_fails"] = 0
           end
           if !state["resumption_token"].nil?
             format = ""
@@ -341,30 +358,39 @@ module OAISchedules
             to = content["until"] || ""
             set = content["set"] || ""
             resumption_token = ""
-            state["count_partial_harversting"] = 0
           end
-          data = oai_get_records(
-            name,
-            content["repository"]["uri"],
-            format,
-            from,
-            to,
-            set,
-            resumption_token
-          )
-          state["resumption_token"] = data["resumptionToken"]
-          state["count_partial_harversting"] += 1
+          data = nil
+          error = nil
+          begin
+            data = oai_get_records(
+              name,
+              content["repository"]["uri"],
+              format,
+              from,
+              to,
+              set,
+              resumption_token
+            )
+            state["resumption_token"] = data["resumptionToken"]
+            state["count_success"] += 1
+          rescue StandardError => e
+            state["count_fails"] += 1
+            error = e
+          end
           path_file_state = get_path_state_file_from_schedule_name(name)
           @logger.info("#{name}: writing to state file #{path_file_state}")
           write_state_file(path_file_state, state)
-          if !data["resumptionToken"].nil?
-            state_machine.add_event(EventHarvesting::DONE_HARVEST)
-            done = false
-          else
-            state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
-            done = true
+          done = false
+          if error.nil?
+            if !data["resumptionToken"].nil?
+              state_machine.add_event(EventHarvesting::DONE_HARVEST)
+              done = false
+            else
+              state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
+              done = true
+            end
           end
-          @f_digest&.call(name, content, data, done)
+          @f_digest&.call(name, content, data, done, error, state, @logger)
           break
         when StateHarvesting::COMPLETE
           @logger.warn("#{name}: full harvesting complete")
@@ -416,7 +442,7 @@ module OAISchedules
       @logger.info("#{name}: fetching from #{url_query}")
       data = DataCollector::Core.filter(DataCollector::Input.new.from_uri(url_query), "$..#{verb}")
       if data.empty?
-        raise StandardError, "#{name}: GET #{url_query}: URL not available, or response has no element #{verb}"
+        raise StandardError, "#{name}: GET #{url_query}: URL not available, or response has no element #{verb}, or missing query parameters (e.g. set)"
       end
       data = data[0]
     rescue DataCollector::InputError => e

data/lib/oai_schedules/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OaiSchedules
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: oai_schedules
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Davide Monari
 bindir: exe
 cert_chain: []
-date: 2025-03-26 00:00:00.000000000 Z
+date: 2025-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -78,6 +78,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json
 - examples/dir_schedules/schedule_sample.json
 - examples/dir_state/.gitkeep
 - examples/example_01.rb