oai_schedules 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d574b21314744d8ad56d1a1394d3fc778e860ba636d43569fedee7642ff3af2
-  data.tar.gz: b06822ea6110eedf0663f40f4c32120c321c0acd9a508deabe3204cb6f094962
+  metadata.gz: 3f254ace930dec7e151a5af485130dd23623548d6f9d0860a764b90ce217c4bb
+  data.tar.gz: bfe93c7f19a894976c53f3d2e3d720e048c77c116b92283034ba7b17d4efe386
 SHA512:
-  metadata.gz: 288bea5932749f50a8629e6fe7ebc041c68f19d39ca44a7513cbdfe1f24682f079f8d8434eabea6ce1cd1d1d3f40f71923ea4bcd701a5926589c544880de2b47
-  data.tar.gz: e2c309f3b419a605ad1eac62023bfdbb4102cd1ae7be8809659a42909851abcb90cfc22bccc790d21f5c43554f744f7bffaa40e006d7f67e91db77b20f50c0e6
+  metadata.gz: bbd9df7881c619f2660dd6b622adf2fd7ec6fa35cde400ff3490b1001b3635c7d89605882e53c9b242cc992de6d0153a7ec5d25d5b41e0672853b997924900a0
+  data.tar.gz: 918414787e68f89b8dab0e692096e708864ca7b32a81d1341c0cc09e72fbb70d2cd886a137f024e5946b6176b7d4b1b20d606ec12d33c81894c65588ca9c712e

data/README.md

@@ -22,8 +22,13 @@ gem install oai_schedules
 ```ruby
 require 'oai_schedules/manager'
 
-f_show = lambda do |name, content, records, done, logger|
-  # ... do your stuff with records ...
+f_show = lambda do |name, content, records, done, error, state, logger|
+  if error.nil?
+    # ... do your stuff with records ...
+  else
+    puts error.message
+  end
+  puts state
   if done
     puts "done full harvesting"
   end
@@ -59,7 +64,8 @@ The code will do this by querying `https://eudml.org/oai/OAIHandler?verb=ListRec
 query parameters.
 The custom function provided as `f_digest` will then be called at each iteration.
 This will be provided schedule `name` and `content`, the partial list of `records` as a hash, 
-a `done` flag (full harvesting complete), and the same logger used internally by the schedules manager.
+a `done` flag (full harvesting complete), an `error` exception if happened, the harvesting `state`,
+and the same logger used internally by the schedules manager.
 and it will write the new one to the state file, until no token is provided (end of the harvesting).
 As soon as the schedule is added, it is executed.
 It is possible to add all schedules in advance, then call `sleep` for infinite event loop.

data/examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json

@@ -0,0 +1,63 @@
+{
+  "interval": "P1W",
+  "repository": {
+    "uri": "https://eudml.org/oai/OAIHandler",
+    "repository_name": "REPOX Repository",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "mailto:gilberto.pedrosa@ist.utl.pt"
+    ],
+    "earliest_datestamp": "1970-01-01T00:00:00+00:00",
+    "deleted_records": "persistent",
+    "granularity": "YYYY-MM-DD",
+    "metadata_format": [
+      "oai_dc",
+      "ese",
+      "eudml-article2",
+      "eudml-book2"
+    ],
+    "set": [
+      "BDIM",
+      "BulDML",
+      "CEDRAM",
+      "DMLE",
+      "DML_CZ_Monograph",
+      "DML_CZ_Proceeding",
+      "DML_CZ_Serial",
+      "EDPS",
+      "ELibM",
+      "GALLICA",
+      "GDZ_Band",
+      "GDZ_Mathematica",
+      "GDZ_Monographs",
+      "GDZ_RusDML",
+      "HDML_Books",
+      "HDML_Conferences",
+      "HDML_Journals",
+      "MISANU",
+      "NUMDAM",
+      "NUMDAM_book",
+      "PLDML",
+      "PLDML_book",
+      "PMath"
+    ],
+    "id": "02df523af427deb93b7cb4600ca347f9297d0e31d51c2783c634459dac457bd0"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "b1671aad-e825-4b5a-b50b-d8591b425e2a",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "7889ce03-28d9-479d-bb9a-b239f179453a"
+  },
+  "format": "oai_dc",
+  "set": "CEDRAM",
+  "from": "1970-01-01T00:00:00+00:00",
+  "until": "1999-01-12T09:43:02+00:00",
+  "id": "dc34623d-2ae2-4e90-91d8-26f4ba29a056"
+}

data/examples/example_01.rb

@@ -4,8 +4,13 @@ require 'oai_schedules/manager'
 
 # usage with folder listener
 
-f_show = lambda do |name, content, records, done, logger|
-  # ... do your stuff with records ...
+f_show = lambda do |name, content, records, done, error, state, logger|
+  if error.nil?
+    # ... do your stuff with records ...
+  else
+    puts error.message
+  end
+  puts state
   if done
     puts "done full harvesting"
   end

data/examples/example_02.rb

@@ -4,8 +4,13 @@ require 'oai_schedules/manager'
 
 # usage with programmatic schedules addition / modify / remove
 
-f_show = lambda do |name, content, records, done, logger|
-  # ... do your stuff with records ...
+f_show = lambda do |name, content, records, done, error, state, logger|
+  if error.nil?
+    # ... do your stuff with records ...
+  else
+    puts error.message
+  end
+  puts state
   if done
     puts "done full harvesting"
   end

data/lib/oai_schedules/manager.rb

@@ -230,7 +230,8 @@ module OAISchedules
       if    (content["format"] != @schedules[name][:content]["format"]) \
         ||  (content["set"] != @schedules[name][:content]["set"])
         state["resumption_token"] = nil
-        state["count_partial_harversting"] = 0
+        state["count_success"] = 0
+        state["count_fails"] = 0
       end
     end
 
@@ -339,8 +340,11 @@ module OAISchedules
         when StateHarvesting::IDLE
           state_machine.add_event(EventHarvesting::REQUEST_HARVEST)
         when StateHarvesting::HARVESTING
-          unless state.has_key?("count_partial_harversting")
-            state["count_partial_harversting"] = 0
+          unless state.has_key?("count_success")
+            state["count_success"] = 0
+          end
+          unless state.has_key?("count_fails")
+            state["count_fails"] = 0
           end
           if !state["resumption_token"].nil?
             format = ""
@@ -354,30 +358,39 @@ module OAISchedules
             to = content["until"] || ""
             set = content["set"] || ""
             resumption_token = ""
-            state["count_partial_harversting"] = 0
           end
-          data = oai_get_records(
-            name,
-            content["repository"]["uri"],
-            format,
-            from,
-            to,
-            set,
-            resumption_token
-          )
-          state["resumption_token"] = data["resumptionToken"]
-          state["count_partial_harversting"] += 1
+          data = nil
+          error = nil
+          begin
+            data = oai_get_records(
+              name,
+              content["repository"]["uri"],
+              format,
+              from,
+              to,
+              set,
+              resumption_token
+            )
+            state["resumption_token"] = data["resumptionToken"]
+            state["count_success"] += 1
+          rescue StandardError => e
+            state["count_fails"] += 1
+            error = e
+          end
           path_file_state = get_path_state_file_from_schedule_name(name)
           @logger.info("#{name}: writing to state file #{path_file_state}")
           write_state_file(path_file_state, state)
-          if !data["resumptionToken"].nil?
-            state_machine.add_event(EventHarvesting::DONE_HARVEST)
-            done = false
-          else
-            state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
-            done = true
+          done = false
+          if error.nil?
+            if !data["resumptionToken"].nil?
+              state_machine.add_event(EventHarvesting::DONE_HARVEST)
+              done = false
+            else
+              state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
+              done = true
+            end
           end
-          @f_digest&.call(name, content, data, done, @logger)
+          @f_digest&.call(name, content, data, done, error, state, @logger)
           break
         when StateHarvesting::COMPLETE
           @logger.warn("#{name}: full harvesting complete")

data/lib/oai_schedules/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OaiSchedules
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: oai_schedules
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Davide Monari
 bindir: exe
 cert_chain: []
-date: 2025-03-28 00:00:00.000000000 Z
+date: 2025-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -78,6 +78,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json
 - examples/dir_schedules/schedule_sample.json
 - examples/dir_state/.gitkeep
 - examples/example_01.rb