oai_schedules 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d574b21314744d8ad56d1a1394d3fc778e860ba636d43569fedee7642ff3af2
-  data.tar.gz: b06822ea6110eedf0663f40f4c32120c321c0acd9a508deabe3204cb6f094962
+  metadata.gz: d2b2bae058157785c775ebd1b8c0ea76e8685adb8588280f6c6427644b9da8b5
+  data.tar.gz: 928102f14abd5bf59bb7b55bb7e100074c5b20bec3276e1dd129a04e74c6da60
 SHA512:
-  metadata.gz: 288bea5932749f50a8629e6fe7ebc041c68f19d39ca44a7513cbdfe1f24682f079f8d8434eabea6ce1cd1d1d3f40f71923ea4bcd701a5926589c544880de2b47
-  data.tar.gz: e2c309f3b419a605ad1eac62023bfdbb4102cd1ae7be8809659a42909851abcb90cfc22bccc790d21f5c43554f744f7bffaa40e006d7f67e91db77b20f50c0e6
+  metadata.gz: 35bb16497f382eb1accaecd95e1e2b022f3addeeef7a4ae79ab5bc4b9f6b023f3fbee70535815ee0edcf5e48b62fc4bdb697cba378cdbac16f114420394cf85e
+  data.tar.gz: 3811fd28be558414ebfabcc83e1978080578ff4b33147f0da06cf569ab6b42af498489cf7bef3dd5f14b6e0dbb8598dc615d15220c7496c710f93fcea0149a3e

data/README.md

@@ -22,8 +22,13 @@ gem install oai_schedules
 ```ruby
 require 'oai_schedules/manager'
 
-f_show = lambda do |name, content, records, done, logger|
-  # ... do your stuff with records ...
+f_show = lambda do |name, content, records, done, error, state, logger|
+  if error.nil?
+    # ... do your stuff with records ...
+  else
+    puts error.message
+  end
+  puts state
   if done
     puts "done full harvesting"
   end
@@ -59,7 +64,8 @@ The code will do this by querying `https://eudml.org/oai/OAIHandler?verb=ListRec
 query parameters.
 The custom function provided as `f_digest` will then be called at each iteration.
 This will be provided schedule `name` and `content`, the partial list of `records` as a hash, 
-a `done` flag (full harvesting complete), and the same logger used internally by the schedules manager.
+a `done` flag (full harvesting complete), an `error` exception if happened, the harvesting `state`,
+and the same logger used internally by the schedules manager.
 and it will write the new one to the state file, until no token is provided (end of the harvesting).
 As soon as the schedule is added, it is executed.
 It is possible to add all schedules in advance, then call `sleep` for infinite event loop.

data/examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json

@@ -0,0 +1,63 @@
+{
+  "interval": "P1W",
+  "repository": {
+    "uri": "https://eudml.org/oai/OAIHandler",
+    "repository_name": "REPOX Repository",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "mailto:gilberto.pedrosa@ist.utl.pt"
+    ],
+    "earliest_datestamp": "1970-01-01T00:00:00+00:00",
+    "deleted_records": "persistent",
+    "granularity": "YYYY-MM-DD",
+    "metadata_format": [
+      "oai_dc",
+      "ese",
+      "eudml-article2",
+      "eudml-book2"
+    ],
+    "set": [
+      "BDIM",
+      "BulDML",
+      "CEDRAM",
+      "DMLE",
+      "DML_CZ_Monograph",
+      "DML_CZ_Proceeding",
+      "DML_CZ_Serial",
+      "EDPS",
+      "ELibM",
+      "GALLICA",
+      "GDZ_Band",
+      "GDZ_Mathematica",
+      "GDZ_Monographs",
+      "GDZ_RusDML",
+      "HDML_Books",
+      "HDML_Conferences",
+      "HDML_Journals",
+      "MISANU",
+      "NUMDAM",
+      "NUMDAM_book",
+      "PLDML",
+      "PLDML_book",
+      "PMath"
+    ],
+    "id": "02df523af427deb93b7cb4600ca347f9297d0e31d51c2783c634459dac457bd0"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "b1671aad-e825-4b5a-b50b-d8591b425e2a",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "7889ce03-28d9-479d-bb9a-b239f179453a"
+  },
+  "format": "oai_dc",
+  "set": "CEDRAM",
+  "from": "1970-01-01T00:00:00+00:00",
+  "until": "1999-01-12T09:43:02+00:00",
+  "id": "dc34623d-2ae2-4e90-91d8-26f4ba29a056"
+}

data/examples/example_01.rb

@@ -4,8 +4,13 @@ require 'oai_schedules/manager'
 
 # usage with folder listener
 
-f_show = lambda do |name, content, records, done, logger|
-  # ... do your stuff with records ...
+f_show = lambda do |name, content, records, done, error, state, logger|
+  if error.nil?
+    # ... do your stuff with records ...
+  else
+    puts error.message
+  end
+  puts state
   if done
     puts "done full harvesting"
   end

data/examples/example_02.rb

@@ -4,8 +4,13 @@ require 'oai_schedules/manager'
 
 # usage with programmatic schedules addition / modify / remove
 
-f_show = lambda do |name, content, records, done, logger|
-  # ... do your stuff with records ...
+f_show = lambda do |name, content, records, done, error, state, logger|
+  if error.nil?
+    # ... do your stuff with records ...
+  else
+    puts error.message
+  end
+  puts state
   if done
     puts "done full harvesting"
   end

data/lib/oai_schedules/manager.rb

@@ -181,6 +181,9 @@ module OAISchedules
       state = {}
       if File.file?(path_file_state)
         state = read_state_file(path_file_state)
+      else
+        init_schedule_state(state)
+        write_state_file(path_file_state, state)
       end
       # create task
       task = Concurrent::TimerTask.new(run_now: false) {
@@ -224,13 +227,22 @@ module OAISchedules
     private
 
 
+    def init_schedule_state(state)
+      state["resumption_token"] = nil
+      state["count_success"] = 0
+      state["count_fails"] = 0
+      state["done"] = false
+      state["count_harvested_records"] = 0
+      state["latest_harvested_records_datestamp"] = ""
+    end
+
+
     def handle_schedule_state_at_schedule_change(name, content)
       # invalidate resumption token if either schedule format or set changes
       state = @schedules[name][:state]
       if    (content["format"] != @schedules[name][:content]["format"]) \
         ||  (content["set"] != @schedules[name][:content]["set"])
-        state["resumption_token"] = nil
-        state["count_partial_harversting"] = 0
+        init_schedule_state(state)
       end
     end
 
@@ -339,8 +351,20 @@ module OAISchedules
         when StateHarvesting::IDLE
           state_machine.add_event(EventHarvesting::REQUEST_HARVEST)
         when StateHarvesting::HARVESTING
-          unless state.has_key?("count_partial_harversting")
-            state["count_partial_harversting"] = 0
+          unless state.has_key?("count_success")
+            state["count_success"] = 0
+          end
+          unless state.has_key?("count_fails")
+            state["count_fails"] = 0
+          end
+          unless state.has_key?("done")
+            state["done"] = false
+          end
+          unless state.has_key?("count_harvested_records")
+            state["count_harvested_records"] = 0
+          end
+          unless state.has_key?("latest_harvested_records_datestamp")
+            state["latest_harvested_records_datestamp"] = ""
           end
           if !state["resumption_token"].nil?
             format = ""
@@ -354,30 +378,53 @@ module OAISchedules
             to = content["until"] || ""
             set = content["set"] || ""
             resumption_token = ""
-            state["count_partial_harversting"] = 0
           end
-          data = oai_get_records(
-            name,
-            content["repository"]["uri"],
-            format,
-            from,
-            to,
-            set,
-            resumption_token
-          )
-          state["resumption_token"] = data["resumptionToken"]
-          state["count_partial_harversting"] += 1
-          path_file_state = get_path_state_file_from_schedule_name(name)
-          @logger.info("#{name}: writing to state file #{path_file_state}")
-          write_state_file(path_file_state, state)
-          if !data["resumptionToken"].nil?
-            state_machine.add_event(EventHarvesting::DONE_HARVEST)
-            done = false
-          else
+          data = nil
+          error = nil
+          done = false
+          if state["done"]
             state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
             done = true
+          else
+            begin
+              data = oai_get_records(
+                name,
+                content["repository"]["uri"],
+                format,
+                from,
+                to,
+                set,
+                resumption_token
+              )
+              state["resumption_token"] = data["resumptionToken"]
+              state["count_success"] += 1
+              n_records = data["record"].size
+              state["count_harvested_records"] += n_records
+              if n_records > 0
+                timestamps = data["record"].map do |record|
+                  record["header"]["datestamp"]
+                end.sort
+                state["latest_harvested_records_datestamp"] = timestamps[-1]
+              end
+            rescue StandardError => e
+              state["count_fails"] += 1
+              error = e
+            end
+            if error.nil?
+              if !data["resumptionToken"].nil?
+                state_machine.add_event(EventHarvesting::DONE_HARVEST)
+                done = false
+              else
+                state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
+                done = true
+              end
+            end
+            state["done"] = done
           end
-          @f_digest&.call(name, content, data, done, @logger)
+          path_file_state = get_path_state_file_from_schedule_name(name)
+          @logger.info("#{name}: writing to state file #{path_file_state}")
+          write_state_file(path_file_state, state)
+          @f_digest&.call(name, content, data, done, error, state, @logger)
           break
         when StateHarvesting::COMPLETE
           @logger.warn("#{name}: full harvesting complete")

data/lib/oai_schedules/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OaiSchedules
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: oai_schedules
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Davide Monari
 bindir: exe
 cert_chain: []
-date: 2025-03-28 00:00:00.000000000 Z
+date: 2025-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -78,6 +78,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json
 - examples/dir_schedules/schedule_sample.json
 - examples/dir_state/.gitkeep
 - examples/example_01.rb