oai_schedules 0.5.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f254ace930dec7e151a5af485130dd23623548d6f9d0860a764b90ce217c4bb
-  data.tar.gz: bfe93c7f19a894976c53f3d2e3d720e048c77c116b92283034ba7b17d4efe386
+  metadata.gz: 93b5d732552fa18fc37680009dd21bf9509555c5cfeec8c26b2b20a37967f606
+  data.tar.gz: cb282dfcd7e8866faa698e5f310ba07223e93d965395734a26665e531cca3ce0
 SHA512:
-  metadata.gz: bbd9df7881c619f2660dd6b622adf2fd7ec6fa35cde400ff3490b1001b3635c7d89605882e53c9b242cc992de6d0153a7ec5d25d5b41e0672853b997924900a0
-  data.tar.gz: 918414787e68f89b8dab0e692096e708864ca7b32a81d1341c0cc09e72fbb70d2cd886a137f024e5946b6176b7d4b1b20d606ec12d33c81894c65588ca9c712e
+  metadata.gz: 54a2044fe8446bf0951415302b68d06dccb3c36fc2628be888681418d1a4cecfcfd2723adbce8dd61656a3081087516e40c28e40acaa9037fdbeda88797f9d28
+  data.tar.gz: 553f101400c60d6f1134f8ea97c075183a595059737605ab49ede6ef8620a5e0b15d47b2f82ff4253036c485489ddebc0059074f3d87bc3c2fc523d98e067600

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## [0.7.0] - 2025-07-09
+
+- Partial harvest policy changed: avoid using resumption tokens
+  (adds complication due to expiration time), and change "from" query param
+  as datestamp of the most recent record fetched + eps (from granularity)
+- Bugfix: "from" and "to" query params now have correct format based on granularity
+- Better exceptions management
+- Some cleanup and sparse bugfixes
+- Added more sample schedules
+- Added more info in the schedule state
+
 ## [0.4.0] - 2025-03-28
 
 - Added logger to digestion function

data/examples/dir_schedules/schedule_495607cf-f773-463f-8ee1-77d0f53e0c29.json

@@ -0,0 +1,38 @@
+{
+  "interval": "PT5S",
+  "repository": {
+    "uri": "https://opac.amsab.be/OAI/Server",
+    "repository_name": "Amsab Online Catalog",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "help@amsab.be"
+    ],
+    "earliest_datestamp": "2000-01-01T00:00:00+00:00",
+    "deleted_records": "transient",
+    "granularity": "YYYY-MM-DDThh:mm:ssZ",
+    "metadata_format": [
+      "oai_dc",
+      "marc21"
+    ],
+    "set": [
+      "collection"
+    ],
+    "id": "490cac7921d1801e6d398aa09bca3ef8f51d07e7dae7cc25676d1ea94126483f"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
+  },
+  "format": "oai_dc",
+  "set": "collection",
+  "from": "2000-01-01T00:00:00+00:00",
+  "id": "495607cf-f773-463f-8ee1-77d0f53e0c29"
+}

data/examples/dir_schedules/schedule_5e344861-806b-4361-98f7-a0be6a5984de.json

@@ -0,0 +1,38 @@
+{
+  "interval": "PT5S",
+  "repository": {
+    "uri": "https://heron.libis.be/ca_veb_q/admin/service.php/OAI/cw_organisaties/request",
+    "repository_name": "VEB",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "collectiveaccess@vlaamse-erfgoedbibliotheken.be"
+    ],
+    "earliest_datestamp": "2020-08-17T12:55:02+00:00",
+    "deleted_records": "transient",
+    "granularity": "YYYY-MM-DDThh:mm:ssZ",
+    "metadata_format": [
+      "oai_dc",
+      "oai_veb"
+    ],
+    "set": [
+      "1"
+    ],
+    "id": "4886e3793003d2b1dffbbdb41ca13024c7c809892314ab301ded0d7ecfd7b469"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
+  },
+  "format": "oai_dc",
+  "set": "1",
+  "from": "2025-06-10T12:39:00+00:00",
+  "id": "5e344861-806b-4361-98f7-a0be6a5984de"
+}

data/examples/dir_schedules/schedule_712f46ab-f87f-4db8-b69e-7101d9e2ae61.json

@@ -0,0 +1,41 @@
+{
+  "interval": "PT5S",
+  "repository": {
+    "uri": "http://lag.hosting.deventit.net/atlantispubliek/oai.axd",
+    "repository_name": "Liberas",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "support@deventit.nl"
+    ],
+    "earliest_datestamp": "1900-01-01T01:01:01+00:00",
+    "deleted_records": "persistent",
+    "granularity": "YYYY-MM-DDThh:mm:ssZ",
+    "metadata_format": [
+      "APEX",
+      "EAC",
+      "EAD"
+    ],
+    "set": [
+      "APEX",
+      "APEXAO",
+      "EAC",
+      "EAD"
+    ],
+    "id": "93b6d240ac9b782664f18823b761a128fc19116506dd228763b52ba8fb64e1b9"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
+  },
+  "format": "EAD",
+  "set": "EAD",
+  "id": "712f46ab-f87f-4db8-b69e-7101d9e2ae61"
+}

data/examples/dir_schedules/schedule_99922363-5b37-4438-a274-5a4a5167f811.json

@@ -0,0 +1,62 @@
+{
+  "interval": "PT5S",
+  "repository": {
+    "uri": "https://eudml.org/oai/OAIHandler",
+    "repository_name": "REPOX Repository",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "mailto:gilberto.pedrosa@ist.utl.pt"
+    ],
+    "earliest_datestamp": "1970-01-01T00:00:00+00:00",
+    "deleted_records": "persistent",
+    "granularity": "YYYY-MM-DD",
+    "metadata_format": [
+      "oai_dc",
+      "ese",
+      "eudml-article2",
+      "eudml-book2"
+    ],
+    "set": [
+      "BDIM",
+      "BulDML",
+      "CEDRAM",
+      "DMLE",
+      "DML_CZ_Monograph",
+      "DML_CZ_Proceeding",
+      "DML_CZ_Serial",
+      "EDPS",
+      "ELibM",
+      "GALLICA",
+      "GDZ_Band",
+      "GDZ_Mathematica",
+      "GDZ_Monographs",
+      "GDZ_RusDML",
+      "HDML_Books",
+      "HDML_Conferences",
+      "HDML_Journals",
+      "MISANU",
+      "NUMDAM",
+      "NUMDAM_book",
+      "PLDML",
+      "PLDML_book",
+      "PMath"
+    ],
+    "id": "02df523af427deb93b7cb4600ca347f9297d0e31d51c2783c634459dac457bd0"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "b1671aad-e825-4b5a-b50b-d8591b425e2a",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "7889ce03-28d9-479d-bb9a-b239f179453a"
+  },
+  "format": "oai_dc",
+  "set": "CEDRAM",
+  "from": "1970-01-01T00:00:00+00:00",
+  "id": "99922363-5b37-4438-a274-5a4a5167f811"
+}

data/examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json

@@ -1,5 +1,5 @@
 {
-  "interval": "P1W",
+  "interval": "PT5S",
   "repository": {
     "uri": "https://eudml.org/oai/OAIHandler",
     "repository_name": "REPOX Repository",
@@ -58,6 +58,5 @@
   "format": "oai_dc",
   "set": "CEDRAM",
   "from": "1970-01-01T00:00:00+00:00",
-  "until": "1999-01-12T09:43:02+00:00",
   "id": "dc34623d-2ae2-4e90-91d8-26f4ba29a056"
 }

data/examples/dir_schedules/schedule_sample.json

@@ -1,10 +1,11 @@
 {
-  "interval": "PT2S",
+  "interval": "PT5S",
   "active": true,
   "repository": {
-    "uri": "https://eu.alma.exlibrisgroup.com/view/oai/32KUL_KUL/request"
+    "uri": "https://eu.alma.exlibrisgroup.com/view/oai/32KUL_KUL/request",
+    "granularity": "YYYY-MM-DDThh:mm:ssZ"
   },
   "format": "marc21",
   "set": "KUL_Rapid_Journals_Print_LendableInternational",
-  "from": "2025-03-23T00:00:00Z"
+  "from": "2024-03-01T00:00:00Z"
 }

data/examples/example_01.rb

@@ -9,8 +9,9 @@ f_show = lambda do |name, content, records, done, error, state, logger|
     # ... do your stuff with records ...
   else
     puts error.message
+    puts error.backtrace
   end
-  puts state
+  puts JSON.pretty_generate(state)
   if done
     puts "done full harvesting"
   end

data/examples/example_02.rb

@@ -9,8 +9,9 @@ f_show = lambda do |name, content, records, done, error, state, logger|
     # ... do your stuff with records ...
   else
     puts error.message
+    puts error.backtrace
   end
-  puts state
+  puts JSON.pretty_generate(state)
   if done
     puts "done full harvesting"
   end

data/lib/oai_schedules/manager.rb

@@ -7,7 +7,7 @@ require 'concurrent-ruby'
 require 'logger'
 require 'data_collector'
 require 'iso8601'
-
+require 'date'
 
 
 
@@ -181,6 +181,9 @@ module OAISchedules
       state = {}
       if File.file?(path_file_state)
         state = read_state_file(path_file_state)
+      else
+        init_schedule_state(state)
+        write_state_file(path_file_state, state)
       end
       # create task
       task = Concurrent::TimerTask.new(run_now: false) {
@@ -224,14 +227,25 @@ module OAISchedules
     private
 
 
+    def init_schedule_state(state)
+      state["resumption_token"] = nil
+      state["expiration_date_resumption_token"] = nil
+      state["datetime_now"] = get_datetime_now
+      state["count_success"] = 0
+      state["count_fails"] = 0
+      state["done"] = false
+      state["count_harvested_records"] = 0
+      state["latest_harvested_records_datestamp"] = nil
+    end
+
+
     def handle_schedule_state_at_schedule_change(name, content)
-      # invalidate resumption token if either schedule format or set changes
       state = @schedules[name][:state]
       if    (content["format"] != @schedules[name][:content]["format"]) \
-        ||  (content["set"] != @schedules[name][:content]["set"])
-        state["resumption_token"] = nil
-        state["count_success"] = 0
-        state["count_fails"] = 0
+        ||  (content["set"] != @schedules[name][:content]["set"]) \
+        ||  (content["from"] != @schedules[name][:content]["from"]) \
+        ||  (content["until"] != @schedules[name][:content]["until"])
+        init_schedule_state(state)
       end
     end
 
@@ -336,62 +350,91 @@ module OAISchedules
           )
           state["identify"] = data
           state_machine.add_event(EventHarvesting::DONE_IDENTIFY)
-          break
         when StateHarvesting::IDLE
           state_machine.add_event(EventHarvesting::REQUEST_HARVEST)
         when StateHarvesting::HARVESTING
-          unless state.has_key?("count_success")
-            state["count_success"] = 0
-          end
-          unless state.has_key?("count_fails")
-            state["count_fails"] = 0
-          end
-          if !state["resumption_token"].nil?
-            format = ""
-            from = ""
-            to = ""
-            set = ""
-            resumption_token = state["resumption_token"]
+          use_resumption_token = false   # don't change
+          format = content["format"] || ""
+          fmt_dt = content.dig("repository", "granularity")
+          from = convert_datetime(content["from"], fmt_dt) || ""
+          to = convert_datetime(content["until"], fmt_dt)  || ""
+          set = content["set"] || ""
+          resumption_token = ""
+          if use_resumption_token
+            if !state["resumption_token"].nil?
+              format = ""
+              from = ""
+              to = ""
+              set = ""
+              resumption_token = state["resumption_token"]
+            end
           else
-            format = content["format"] || ""
-            from = content["from"] || ""
-            to = content["until"] || ""
-            set = content["set"] || ""
-            resumption_token = ""
+            from = add_eps_to_datetime(state["latest_harvested_records_datestamp"], fmt_dt) || from
           end
           data = nil
           error = nil
-          begin
-            data = oai_get_records(
-              name,
-              content["repository"]["uri"],
-              format,
-              from,
-              to,
-              set,
-              resumption_token
-            )
-            state["resumption_token"] = data["resumptionToken"]
-            state["count_success"] += 1
-          rescue StandardError => e
-            state["count_fails"] += 1
-            error = e
+          to_pause = true
+          done = false
+          if state["done"]
+            state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
+            done = true
+          else
+            begin
+              data = oai_get_records(
+                name,
+                content["repository"]["uri"],
+                format,
+                from,
+                to,
+                set,
+                resumption_token
+              )
+              state["resumption_token"] = nil
+              state["expiration_date_resumption_token"] = nil
+              if use_resumption_token
+                if data["resumptionToken"].is_a?(String)
+                  state["resumption_token"] = data["resumptionToken"]
+                elsif data["resumptionToken"].is_a?(Hash)
+                  data_token = data["resumptionToken"]
+                  state["resumption_token"] = data_token["$text"]
+                  state["expiration_date_resumption_token"] = data_token["_expirationDate"]
+                end
+              end
+              state["datetime_now"] = get_datetime_now
+              n_records = data["record"].size
+              state["count_harvested_records"] += n_records
+              if n_records > 0
+                timestamps = data["record"].map do |record|
+                  record["header"]["datestamp"]
+                end.sort
+                timestamp_latest = timestamps[-1].strftime('%FT%TZ')
+                state["latest_harvested_records_datestamp"] = convert_datetime(timestamp_latest, fmt_dt)
+              end
+              state["count_success"] += 1
+            rescue StandardError => e
+              state["count_fails"] += 1
+              error = e
+            end
+            if error.nil?
+              if use_resumption_token
+                if state["resumption_token"].nil?
+                  state_machine.add_event(EventHarvesting::DONE_HARVEST)
+                  to_pause = true
+                else
+                  to_pause = false
+                end
+              else
+                state_machine.add_event(EventHarvesting::DONE_HARVEST)
+                to_pause = true
+              end
+            end
+            state["done"] = done
           end
           path_file_state = get_path_state_file_from_schedule_name(name)
           @logger.info("#{name}: writing to state file #{path_file_state}")
           write_state_file(path_file_state, state)
-          done = false
-          if error.nil?
-            if !data["resumptionToken"].nil?
-              state_machine.add_event(EventHarvesting::DONE_HARVEST)
-              done = false
-            else
-              state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
-              done = true
-            end
-          end
           @f_digest&.call(name, content, data, done, error, state, @logger)
-          break
+          break if to_pause
         when StateHarvesting::COMPLETE
           @logger.warn("#{name}: full harvesting complete")
           content["active"] = false
@@ -409,11 +452,19 @@ module OAISchedules
       verb = "Identify"
       url_query = "#{url_base}?verb=#{verb}"
       @logger.info("#{name}: fetching from #{url_query}")
-      data = DataCollector::Core.filter(DataCollector::Input.new.from_uri(url_query), "$..#{verb}")
+      input = DataCollector::Input.new.from_uri(url_query)
+      if input.nil?
+        raise StandardError, "#{url_query}, service not found"
+      end
+      data = DataCollector::Core.filter(input, "$..#{verb}")
       if data.empty?
-        raise StandardError, "#{name}: GET #{url_query}: URL not available, or response has no element #{verb}"
+        error = DataCollector::Core.filter(input, "$..error")
+        error = error[0]
+        name_error = error['_code']
+        text_error = error['$text']
+        raise StandardError, "#{name}: #{url_query}: #{name_error}, #{text_error}"
       end
-      data = data[0]
+      data[0]
     rescue DataCollector::InputError => e
       raise RuntimeError, "#{name}: #{url_query} not found, or server error"
     rescue StandardError => e
@@ -440,11 +491,22 @@ module OAISchedules
         url_query += "&resumptionToken=#{resumption_token}"
       end
       @logger.info("#{name}: fetching from #{url_query}")
-      data = DataCollector::Core.filter(DataCollector::Input.new.from_uri(url_query), "$..#{verb}")
+      input = DataCollector::Input.new.from_uri(url_query)
+      if input.nil?
+        raise StandardError, "#{url_query}, service not found"
+      end
+      data = DataCollector::Core.filter(input, "$..#{verb}")
       if data.empty?
-        raise StandardError, "#{name}: GET #{url_query}: URL not available, or response has no element #{verb}, or missing query parameters (e.g. set)"
+        error = DataCollector::Core.filter(input, "$..error")
+        error = error[0]
+        name_error = error['_code']
+        text_error = error['$text']
+        raise StandardError, "#{name}: #{url_query}: #{name_error}, #{text_error}"
       end
-      data = data[0]
+      if data.compact.empty?
+        raise StandardError, "#{name}: #{url_query}: no records found"
+      end
+      data[0]
     rescue DataCollector::InputError => e
       raise RuntimeError, "#{name}: #{url_query} not found, or server error"
     rescue StandardError => e
@@ -477,6 +539,37 @@ module OAISchedules
       modify_schedule(name_schedule, content_schedule)
     end
 
+    def convert_datetime(str_dt, fmt_dt)
+      return nil unless str_dt
+      return str_dt unless fmt_dt
+      dt = DateTime.parse(str_dt)
+      case fmt_dt
+      when 'YYYY-MM-DD'
+        dt.strftime('%F')
+      when 'YYYY-MM-DDThh:mm:ssZ'
+        dt.strftime('%FT%TZ')
+      else
+        str_dt
+      end
+    end
+
+    def add_eps_to_datetime(str_dt, fmt_dt)
+      return nil unless str_dt
+      dt = DateTime.parse(str_dt)
+      case fmt_dt
+      when 'YYYY-MM-DD'
+        dt += 1.0
+        dt.strftime('%F')
+      when 'YYYY-MM-DDThh:mm:ssZ'
+        dt += 1.0/(24*60*60)
+        dt.strftime('%FT%TZ')
+      end
+    end
+
+    def get_datetime_now
+      DateTime.now.strftime('%FT%TZ')
+    end
+
   end
 
 end

data/lib/oai_schedules/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OaiSchedules
-  VERSION = "0.5.0"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: oai_schedules
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Davide Monari
 bindir: exe
 cert_chain: []
-date: 2025-04-29 00:00:00.000000000 Z
+date: 2025-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -78,6 +78,10 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/dir_schedules/schedule_495607cf-f773-463f-8ee1-77d0f53e0c29.json
+- examples/dir_schedules/schedule_5e344861-806b-4361-98f7-a0be6a5984de.json
+- examples/dir_schedules/schedule_712f46ab-f87f-4db8-b69e-7101d9e2ae61.json
+- examples/dir_schedules/schedule_99922363-5b37-4438-a274-5a4a5167f811.json
 - examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json
 - examples/dir_schedules/schedule_sample.json
 - examples/dir_state/.gitkeep