oai_schedules 0.6.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2b2bae058157785c775ebd1b8c0ea76e8685adb8588280f6c6427644b9da8b5
-  data.tar.gz: 928102f14abd5bf59bb7b55bb7e100074c5b20bec3276e1dd129a04e74c6da60
+  metadata.gz: 86d2572fbf72c1d75ab9b83efcbb739726962127953458f438cf5ac20278997d
+  data.tar.gz: b4fe62d6a8280e9bd55c2337d10615768808d0d5ccbeef469a8ad4a7a55b0b05
 SHA512:
-  metadata.gz: 35bb16497f382eb1accaecd95e1e2b022f3addeeef7a4ae79ab5bc4b9f6b023f3fbee70535815ee0edcf5e48b62fc4bdb697cba378cdbac16f114420394cf85e
-  data.tar.gz: 3811fd28be558414ebfabcc83e1978080578ff4b33147f0da06cf569ab6b42af498489cf7bef3dd5f14b6e0dbb8598dc615d15220c7496c710f93fcea0149a3e
+  metadata.gz: 1294b5a37cf3d9ef098c5d1484d1853e5c5990148348b735c7a22809cbab963c1d22e1fae8b3ab525107a21368483ac8eb600f954293dde41a6039db4fed6097
+  data.tar.gz: bc8a60e47cd6f1778483556e644362e976b97456b5c30ae6c320b526b55958e2852011c002c3268afd15c003f835a03b847d0a9173f4590e418ccfece198f447

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## [0.8.0] - 2025-07-10
+
+- Complete harvesting is now within a single thread tick
+- Added more info in the schedule state
+
+## [0.7.0] - 2025-07-09
+
+- Partial harvest policy changed: avoid using resumption tokens
+  (adds complication due to expiration time), and change "from" query param
+  as datestamp of the most recent record fetched + eps (from granularity)
+- Bugfix: "from" and "to" query params now have correct format based on granularity
+- Better exceptions management
+- Some cleanup and sparse bugfixes
+- Added more sample schedules
+- Added more info in the schedule state
+
 ## [0.4.0] - 2025-03-28
 
 - Added logger to digestion function

data/examples/dir_schedules/schedule_495607cf-f773-463f-8ee1-77d0f53e0c29.json

@@ -0,0 +1,38 @@
+{
+  "interval": "PT5S",
+  "repository": {
+    "uri": "https://opac.amsab.be/OAI/Server",
+    "repository_name": "Amsab Online Catalog",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "help@amsab.be"
+    ],
+    "earliest_datestamp": "2000-01-01T00:00:00+00:00",
+    "deleted_records": "transient",
+    "granularity": "YYYY-MM-DDThh:mm:ssZ",
+    "metadata_format": [
+      "oai_dc",
+      "marc21"
+    ],
+    "set": [
+      "collection"
+    ],
+    "id": "490cac7921d1801e6d398aa09bca3ef8f51d07e7dae7cc25676d1ea94126483f"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
+  },
+  "format": "oai_dc",
+  "set": "collection",
+  "from": "2000-01-01T00:00:00+00:00",
+  "id": "495607cf-f773-463f-8ee1-77d0f53e0c29"
+}

data/examples/dir_schedules/schedule_5e344861-806b-4361-98f7-a0be6a5984de.json

@@ -0,0 +1,38 @@
+{
+  "interval": "PT5S",
+  "repository": {
+    "uri": "https://heron.libis.be/ca_veb_q/admin/service.php/OAI/cw_organisaties/request",
+    "repository_name": "VEB",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "collectiveaccess@vlaamse-erfgoedbibliotheken.be"
+    ],
+    "earliest_datestamp": "2020-08-17T12:55:02+00:00",
+    "deleted_records": "transient",
+    "granularity": "YYYY-MM-DDThh:mm:ssZ",
+    "metadata_format": [
+      "oai_dc",
+      "oai_veb"
+    ],
+    "set": [
+      "1"
+    ],
+    "id": "4886e3793003d2b1dffbbdb41ca13024c7c809892314ab301ded0d7ecfd7b469"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
+  },
+  "format": "oai_dc",
+  "set": "1",
+  "from": "2025-06-10T12:39:00+00:00",
+  "id": "5e344861-806b-4361-98f7-a0be6a5984de"
+}

data/examples/dir_schedules/schedule_712f46ab-f87f-4db8-b69e-7101d9e2ae61.json

@@ -0,0 +1,41 @@
+{
+  "interval": "PT5S",
+  "repository": {
+    "uri": "http://lag.hosting.deventit.net/atlantispubliek/oai.axd",
+    "repository_name": "Liberas",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "support@deventit.nl"
+    ],
+    "earliest_datestamp": "1900-01-01T01:01:01+00:00",
+    "deleted_records": "persistent",
+    "granularity": "YYYY-MM-DDThh:mm:ssZ",
+    "metadata_format": [
+      "APEX",
+      "EAC",
+      "EAD"
+    ],
+    "set": [
+      "APEX",
+      "APEXAO",
+      "EAC",
+      "EAD"
+    ],
+    "id": "93b6d240ac9b782664f18823b761a128fc19116506dd228763b52ba8fb64e1b9"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
+  },
+  "format": "EAD",
+  "set": "EAD",
+  "id": "712f46ab-f87f-4db8-b69e-7101d9e2ae61"
+}

data/examples/dir_schedules/schedule_99922363-5b37-4438-a274-5a4a5167f811.json

@@ -0,0 +1,62 @@
+{
+  "interval": "PT5S",
+  "repository": {
+    "uri": "https://eudml.org/oai/OAIHandler",
+    "repository_name": "REPOX Repository",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "mailto:gilberto.pedrosa@ist.utl.pt"
+    ],
+    "earliest_datestamp": "1970-01-01T00:00:00+00:00",
+    "deleted_records": "persistent",
+    "granularity": "YYYY-MM-DD",
+    "metadata_format": [
+      "oai_dc",
+      "ese",
+      "eudml-article2",
+      "eudml-book2"
+    ],
+    "set": [
+      "BDIM",
+      "BulDML",
+      "CEDRAM",
+      "DMLE",
+      "DML_CZ_Monograph",
+      "DML_CZ_Proceeding",
+      "DML_CZ_Serial",
+      "EDPS",
+      "ELibM",
+      "GALLICA",
+      "GDZ_Band",
+      "GDZ_Mathematica",
+      "GDZ_Monographs",
+      "GDZ_RusDML",
+      "HDML_Books",
+      "HDML_Conferences",
+      "HDML_Journals",
+      "MISANU",
+      "NUMDAM",
+      "NUMDAM_book",
+      "PLDML",
+      "PLDML_book",
+      "PMath"
+    ],
+    "id": "02df523af427deb93b7cb4600ca347f9297d0e31d51c2783c634459dac457bd0"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "b1671aad-e825-4b5a-b50b-d8591b425e2a",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "7889ce03-28d9-479d-bb9a-b239f179453a"
+  },
+  "format": "oai_dc",
+  "set": "CEDRAM",
+  "from": "1970-01-01T00:00:00+00:00",
+  "id": "99922363-5b37-4438-a274-5a4a5167f811"
+}

data/examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json

@@ -1,5 +1,5 @@
 {
-  "interval": "P1W",
+  "interval": "PT5S",
   "repository": {
     "uri": "https://eudml.org/oai/OAIHandler",
     "repository_name": "REPOX Repository",
@@ -58,6 +58,5 @@
   "format": "oai_dc",
   "set": "CEDRAM",
   "from": "1970-01-01T00:00:00+00:00",
-  "until": "1999-01-12T09:43:02+00:00",
   "id": "dc34623d-2ae2-4e90-91d8-26f4ba29a056"
 }

data/examples/dir_schedules/schedule_sample.json

@@ -1,10 +1,11 @@
 {
-  "interval": "PT2S",
+  "interval": "PT5S",
   "active": true,
   "repository": {
-    "uri": "https://eu.alma.exlibrisgroup.com/view/oai/32KUL_KUL/request"
+    "uri": "https://eu.alma.exlibrisgroup.com/view/oai/32KUL_KUL/request",
+    "granularity": "YYYY-MM-DDThh:mm:ssZ"
   },
   "format": "marc21",
   "set": "KUL_Rapid_Journals_Print_LendableInternational",
-  "from": "2025-03-23T00:00:00Z"
+  "from": "2024-03-01T00:00:00Z"
 }

data/examples/example_01.rb

@@ -9,8 +9,9 @@ f_show = lambda do |name, content, records, done, error, state, logger|
     # ... do your stuff with records ...
   else
     puts error.message
+    puts error.backtrace
   end
-  puts state
+  puts JSON.pretty_generate(state)
   if done
     puts "done full harvesting"
   end

data/examples/example_02.rb

@@ -9,8 +9,9 @@ f_show = lambda do |name, content, records, done, error, state, logger|
     # ... do your stuff with records ...
   else
     puts error.message
+    puts error.backtrace
   end
-  puts state
+  puts JSON.pretty_generate(state)
   if done
     puts "done full harvesting"
   end

data/lib/oai_schedules/manager.rb

@@ -7,7 +7,7 @@ require 'concurrent-ruby'
 require 'logger'
 require 'data_collector'
 require 'iso8601'
-
+require 'date'
 
 
 
@@ -229,19 +229,25 @@ module OAISchedules
 
     def init_schedule_state(state)
       state["resumption_token"] = nil
+      state["expiration_date_resumption_token"] = nil
+      state["datetime_now"] = get_datetime_now
+      state["datetime_next_harvesting"] = nil
       state["count_success"] = 0
       state["count_fails"] = 0
       state["done"] = false
       state["count_harvested_records"] = 0
-      state["latest_harvested_records_datestamp"] = ""
+      state["latest_harvested_records_datestamp"] = nil
+      state["harvesting"] = false
+      state["error"] = nil
     end
 
 
     def handle_schedule_state_at_schedule_change(name, content)
-      # invalidate resumption token if either schedule format or set changes
       state = @schedules[name][:state]
       if    (content["format"] != @schedules[name][:content]["format"]) \
-        ||  (content["set"] != @schedules[name][:content]["set"])
+        ||  (content["set"] != @schedules[name][:content]["set"]) \
+        ||  (content["from"] != @schedules[name][:content]["from"]) \
+        ||  (content["until"] != @schedules[name][:content]["until"])
         init_schedule_state(state)
       end
     end
@@ -256,8 +262,7 @@ module OAISchedules
       task = @schedules[name][:task]
       interval_s_safe = 60
       begin
-        duration = ISO8601::Duration.new(@schedules[name][:content]["interval"])
-        interval_s = duration.to_seconds
+        interval_s = interval_iso8601_to_seconds(@schedules[name][:content]["interval"])
         @logger.info("#{name}: task interval (s): #{interval_s}")
         th_interval_s = 0.1   # protects from negative, 0 or small time intervals
         if interval_s < th_interval_s
@@ -335,7 +340,9 @@ module OAISchedules
 
 
     def logic(name, content, state_machine, state)
+      state["datetime_next_harvesting"] = get_datetime_next_schedule_tick_from_now(content["interval"])
       loop do
+        # sleep(3)
         @logger.info("#{name}: handling state: #{state_machine.state}")
         case state_machine.state
         when StateHarvesting::NOT_IDENTIFIED
@@ -347,40 +354,31 @@ module OAISchedules
           )
           state["identify"] = data
           state_machine.add_event(EventHarvesting::DONE_IDENTIFY)
-          break
         when StateHarvesting::IDLE
           state_machine.add_event(EventHarvesting::REQUEST_HARVEST)
         when StateHarvesting::HARVESTING
-          unless state.has_key?("count_success")
-            state["count_success"] = 0
-          end
-          unless state.has_key?("count_fails")
-            state["count_fails"] = 0
-          end
-          unless state.has_key?("done")
-            state["done"] = false
-          end
-          unless state.has_key?("count_harvested_records")
-            state["count_harvested_records"] = 0
-          end
-          unless state.has_key?("latest_harvested_records_datestamp")
-            state["latest_harvested_records_datestamp"] = ""
-          end
-          if !state["resumption_token"].nil?
-            format = ""
-            from = ""
-            to = ""
-            set = ""
-            resumption_token = state["resumption_token"]
+          use_resumption_token = false # don't change
+          state["harvesting"] = true
+          format = content["format"] || ""
+          fmt_dt = content.dig("repository", "granularity")
+          from = convert_datetime(content["from"], fmt_dt) || ""
+          to = convert_datetime(content["until"], fmt_dt)  || ""
+          set = content["set"] || ""
+          resumption_token = ""
+          if use_resumption_token
+            if !state["resumption_token"].nil?
+              format = ""
+              from = ""
+              to = ""
+              set = ""
+              resumption_token = state["resumption_token"]
+            end
           else
-            format = content["format"] || ""
-            from = content["from"] || ""
-            to = content["until"] || ""
-            set = content["set"] || ""
-            resumption_token = ""
+            from = add_eps_to_datetime(state["latest_harvested_records_datestamp"], fmt_dt) || from
           end
           data = nil
           error = nil
+          to_pause = true
           done = false
           if state["done"]
             state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
@@ -396,36 +394,60 @@ module OAISchedules
                 set,
                 resumption_token
               )
-              state["resumption_token"] = data["resumptionToken"]
-              state["count_success"] += 1
+              state["resumption_token"] = nil
+              state["expiration_date_resumption_token"] = nil
+              if use_resumption_token
+                if data["resumptionToken"].is_a?(String)
+                  state["resumption_token"] = data["resumptionToken"]
+                elsif data["resumptionToken"].is_a?(Hash)
+                  data_token = data["resumptionToken"]
+                  state["resumption_token"] = data_token["$text"]
+                  state["expiration_date_resumption_token"] = data_token["_expirationDate"]
+                end
+              end
               n_records = data["record"].size
               state["count_harvested_records"] += n_records
               if n_records > 0
                 timestamps = data["record"].map do |record|
                   record["header"]["datestamp"]
                 end.sort
-                state["latest_harvested_records_datestamp"] = timestamps[-1]
+                timestamp_latest = timestamps[-1].strftime('%FT%TZ')
+                state["latest_harvested_records_datestamp"] = convert_datetime(timestamp_latest, fmt_dt)
               end
+              state["count_success"] += 1
             rescue StandardError => e
               state["count_fails"] += 1
               error = e
             end
             if error.nil?
-              if !data["resumptionToken"].nil?
-                state_machine.add_event(EventHarvesting::DONE_HARVEST)
-                done = false
+              state["error"] = nil
+              if use_resumption_token
+                if state["resumption_token"].nil?
+                  state_machine.add_event(EventHarvesting::DONE_HARVEST)
+                  to_pause = true
+                else
+                  to_pause = false
+                end
               else
-                state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
-                done = true
+                to_pause = false
               end
+            else
+              state["error"] = {
+                "message" => error.message,
+                "backtrace" => error.backtrace
+              }
+              state_machine.add_event(EventHarvesting::DONE_HARVEST)
+              to_pause = true
             end
             state["done"] = done
+            state["harvesting"] = !to_pause
+            state["datetime_now"] = get_datetime_now
           end
           path_file_state = get_path_state_file_from_schedule_name(name)
           @logger.info("#{name}: writing to state file #{path_file_state}")
           write_state_file(path_file_state, state)
           @f_digest&.call(name, content, data, done, error, state, @logger)
-          break
+          break if to_pause
         when StateHarvesting::COMPLETE
           @logger.warn("#{name}: full harvesting complete")
           content["active"] = false
@@ -443,11 +465,19 @@ module OAISchedules
       verb = "Identify"
       url_query = "#{url_base}?verb=#{verb}"
       @logger.info("#{name}: fetching from #{url_query}")
-      data = DataCollector::Core.filter(DataCollector::Input.new.from_uri(url_query), "$..#{verb}")
+      input = DataCollector::Input.new.from_uri(url_query)
+      if input.nil?
+        raise StandardError, "#{url_query}, service not found"
+      end
+      data = DataCollector::Core.filter(input, "$..#{verb}")
       if data.empty?
-        raise StandardError, "#{name}: GET #{url_query}: URL not available, or response has no element #{verb}"
+        error = DataCollector::Core.filter(input, "$..error")
+        error = error[0]
+        name_error = error['_code']
+        text_error = error['$text']
+        raise StandardError, "#{name}: #{url_query}: #{name_error}, #{text_error}"
       end
-      data = data[0]
+      data[0]
     rescue DataCollector::InputError => e
       raise RuntimeError, "#{name}: #{url_query} not found, or server error"
     rescue StandardError => e
@@ -474,11 +504,22 @@ module OAISchedules
         url_query += "&resumptionToken=#{resumption_token}"
       end
       @logger.info("#{name}: fetching from #{url_query}")
-      data = DataCollector::Core.filter(DataCollector::Input.new.from_uri(url_query), "$..#{verb}")
+      input = DataCollector::Input.new.from_uri(url_query)
+      if input.nil?
+        raise StandardError, "#{url_query}, service not found"
+      end
+      data = DataCollector::Core.filter(input, "$..#{verb}")
       if data.empty?
-        raise StandardError, "#{name}: GET #{url_query}: URL not available, or response has no element #{verb}, or missing query parameters (e.g. set)"
+        error = DataCollector::Core.filter(input, "$..error")
+        error = error[0]
+        name_error = error['_code']
+        text_error = error['$text']
+        raise StandardError, "#{name}: #{url_query}: #{name_error}, #{text_error}"
       end
-      data = data[0]
+      if data.compact.empty?
+        raise StandardError, "#{name}: #{url_query}: no records found"
+      end
+      data[0]
     rescue DataCollector::InputError => e
       raise RuntimeError, "#{name}: #{url_query} not found, or server error"
     rescue StandardError => e
@@ -511,6 +552,48 @@ module OAISchedules
       modify_schedule(name_schedule, content_schedule)
     end
 
+    def convert_datetime(str_dt, fmt_dt)
+      return nil unless str_dt
+      return str_dt unless fmt_dt
+      dt = DateTime.parse(str_dt)
+      case fmt_dt
+      when 'YYYY-MM-DD'
+        dt.strftime('%F')
+      when 'YYYY-MM-DDThh:mm:ssZ'
+        dt.strftime('%FT%TZ')
+      else
+        str_dt
+      end
+    end
+
+    def add_eps_to_datetime(str_dt, fmt_dt)
+      return nil unless str_dt
+      dt = DateTime.parse(str_dt)
+      case fmt_dt
+      when 'YYYY-MM-DD'
+        dt += 1.0
+        dt.strftime('%F')
+      when 'YYYY-MM-DDThh:mm:ssZ'
+        dt += 1.0/(24*60*60)
+        dt.strftime('%FT%TZ')
+      end
+    end
+
+    def get_datetime_now
+      DateTime.now.strftime('%FT%TZ')
+    end
+
+    def interval_iso8601_to_seconds(str_interval)
+      ISO8601::Duration.new(str_interval).to_seconds
+    end
+
+    def get_datetime_next_schedule_tick_from_now(str_interval)
+      interval_s = interval_iso8601_to_seconds(str_interval)
+      dt = DateTime.now
+      dt += 1.0*interval_s/(24*60*60)
+      dt.strftime('%FT%TZ')
+    end
+
   end
 
 end

data/lib/oai_schedules/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OaiSchedules
-  VERSION = "0.6.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: oai_schedules
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Davide Monari
 bindir: exe
 cert_chain: []
-date: 2025-04-29 00:00:00.000000000 Z
+date: 2025-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -78,6 +78,10 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/dir_schedules/schedule_495607cf-f773-463f-8ee1-77d0f53e0c29.json
+- examples/dir_schedules/schedule_5e344861-806b-4361-98f7-a0be6a5984de.json
+- examples/dir_schedules/schedule_712f46ab-f87f-4db8-b69e-7101d9e2ae61.json
+- examples/dir_schedules/schedule_99922363-5b37-4438-a274-5a4a5167f811.json
 - examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json
 - examples/dir_schedules/schedule_sample.json
 - examples/dir_state/.gitkeep