oai_schedules 0.7.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93b5d732552fa18fc37680009dd21bf9509555c5cfeec8c26b2b20a37967f606
-  data.tar.gz: cb282dfcd7e8866faa698e5f310ba07223e93d965395734a26665e531cca3ce0
+  metadata.gz: dac0dec1b24093321fea8bb4e8fe05a315255d4a4418f0ff72803f6af902196f
+  data.tar.gz: 590c1f29d049ee82a4352ec3d5a79fbd070013be36f4f08a166cf34323c0afd1
 SHA512:
-  metadata.gz: 54a2044fe8446bf0951415302b68d06dccb3c36fc2628be888681418d1a4cecfcfd2723adbce8dd61656a3081087516e40c28e40acaa9037fdbeda88797f9d28
-  data.tar.gz: 553f101400c60d6f1134f8ea97c075183a595059737605ab49ede6ef8620a5e0b15d47b2f82ff4253036c485489ddebc0059074f3d87bc3c2fc523d98e067600
+  metadata.gz: e7fa2c85188069c4997fe4fb91e39f05b6f9ec250184b60868729701a1fefd16fa3fa830cf993913454e5acb8ea4649d2b16a4bad6fcee499053dbe929b5b105
+  data.tar.gz: 50b0ed8e1955e52fe4ea25bd66a38abd397be64d2986ee608c042502091b44e58835110fc18d5406c1c28c431c6e741455091a824ce592c7e8bce4fead953c34

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## [0.9.0] - 2025-07-15
+
+- Restored resumption token logic (the only one correct)
+- Fixed Zulu time issues
+- Added more info in the schedule state
+
+## [0.8.0] - 2025-07-10
+
+- Complete harvesting is now within a single thread tick
+- Added more info in the schedule state
+
 ## [0.7.0] - 2025-07-09
 
 - Partial harvest policy changed: avoid using resumption tokens

data/examples/dir_schedules_all/schedule_deventit.json

@@ -0,0 +1,41 @@
+{
+  "interval": "PT5S",
+  "repository": {
+    "uri": "http://lag.hosting.deventit.net/atlantispubliek/oai.axd",
+    "repository_name": "Liberas",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "support@deventit.nl"
+    ],
+    "earliest_datestamp": "1900-01-01T01:01:01+00:00",
+    "deleted_records": "persistent",
+    "granularity": "YYYY-MM-DDThh:mm:ssZ",
+    "metadata_format": [
+      "APEX",
+      "EAC",
+      "EAD"
+    ],
+    "set": [
+      "APEX",
+      "APEXAO",
+      "EAC",
+      "EAD"
+    ],
+    "id": "93b6d240ac9b782664f18823b761a128fc19116506dd228763b52ba8fb64e1b9"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
+  },
+  "format": "EAD",
+  "set": "EAD",
+  "id": "deventit"
+}

data/examples/dir_schedules_all/schedule_heron.json

@@ -0,0 +1,38 @@
+{
+  "interval": "PT5S",
+  "repository": {
+    "uri": "https://heron.libis.be/ca_veb_q/admin/service.php/OAI/cw_organisaties/request",
+    "repository_name": "VEB",
+    "protocol_version": "2.0",
+    "admin_email": [
+      "collectiveaccess@vlaamse-erfgoedbibliotheken.be"
+    ],
+    "earliest_datestamp": "2020-08-17T12:55:02+00:00",
+    "deleted_records": "transient",
+    "granularity": "YYYY-MM-DDThh:mm:ssZ",
+    "metadata_format": [
+      "oai_dc",
+      "oai_veb"
+    ],
+    "set": [
+      "1"
+    ],
+    "id": "4886e3793003d2b1dffbbdb41ca13024c7c809892314ab301ded0d7ecfd7b469"
+  },
+  "active": true,
+  "transformer": {
+    "transformer_name": [
+      "dummy_transformer"
+    ],
+    "type": {
+      "id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
+      "value": "dummy_type"
+    },
+    "uri": "http://dummy-uri.org/",
+    "id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
+  },
+  "format": "oai_veb",
+  "set": "1",
+  "from": "1900-06-10T12:39:00+00:00",
+  "id": "heron"
+}

data/lib/oai_schedules/manager.rb

@@ -8,6 +8,7 @@ require 'logger'
 require 'data_collector'
 require 'iso8601'
 require 'date'
+require 'active_support/core_ext/time'
 
 
 
@@ -49,6 +50,7 @@ module OAISchedules
       REQUEST_HARVEST = 2
       DONE_HARVEST = 3
       DONE_FULL_HARVEST = 4
+      RESTART = 5
     end
 
     class StateMachineHarvesting
@@ -103,6 +105,15 @@ module OAISchedules
             @state
           end
 
+        when StateHarvesting::COMPLETE
+
+          case event
+          when EventHarvesting::RESTART
+            @state = StateHarvesting::NOT_IDENTIFIED
+          else
+            @state
+          end
+
         else
           @state
         end
@@ -228,14 +239,19 @@ module OAISchedules
 
 
     def init_schedule_state(state)
+      state["use_resumption_token"] = nil
       state["resumption_token"] = nil
       state["expiration_date_resumption_token"] = nil
       state["datetime_now"] = get_datetime_now
+      state["datetime_next_harvesting"] = nil
       state["count_success"] = 0
       state["count_fails"] = 0
       state["done"] = false
       state["count_harvested_records"] = 0
       state["latest_harvested_records_datestamp"] = nil
+      state["harvesting"] = false
+      state["error"] = nil
+      state["status"] = nil
     end
 
 
@@ -259,8 +275,7 @@ module OAISchedules
       task = @schedules[name][:task]
       interval_s_safe = 60
       begin
-        duration = ISO8601::Duration.new(@schedules[name][:content]["interval"])
-        interval_s = duration.to_seconds
+        interval_s = interval_iso8601_to_seconds(@schedules[name][:content]["interval"])
         @logger.info("#{name}: task interval (s): #{interval_s}")
         th_interval_s = 0.1   # protects from negative, 0 or small time intervals
         if interval_s < th_interval_s
@@ -338,7 +353,9 @@ module OAISchedules
 
 
     def logic(name, content, state_machine, state)
+      state["datetime_next_harvesting"] = get_datetime_next_schedule_tick_from_now(content["interval"])
       loop do
+        # sleep(3)
         @logger.info("#{name}: handling state: #{state_machine.state}")
         case state_machine.state
         when StateHarvesting::NOT_IDENTIFIED
@@ -353,7 +370,9 @@ module OAISchedules
         when StateHarvesting::IDLE
           state_machine.add_event(EventHarvesting::REQUEST_HARVEST)
         when StateHarvesting::HARVESTING
-          use_resumption_token = false   # don't change
+          # use_resumption_token = false
+          use_resumption_token = true
+          state["use_resumption_token"] = use_resumption_token
           format = content["format"] || ""
           fmt_dt = content.dig("repository", "granularity")
           from = convert_datetime(content["from"], fmt_dt) || ""
@@ -362,83 +381,101 @@ module OAISchedules
           resumption_token = ""
           if use_resumption_token
             if !state["resumption_token"].nil?
-              format = ""
-              from = ""
-              to = ""
-              set = ""
-              resumption_token = state["resumption_token"]
+              # if state["expiration_date_resumption_token"].nil? || (!state["expiration_date_resumption_token"].nil? && (DateTime.parse(state["expiration_date_resumption_token"]) - DateTime.now) > 0)
+                format = ""
+                from = ""
+                to = ""
+                set = ""
+                resumption_token = state["resumption_token"]
+              # end
             end
           else
             from = add_eps_to_datetime(state["latest_harvested_records_datestamp"], fmt_dt) || from
           end
           data = nil
           error = nil
-          to_pause = true
+          err_info = nil
+          harvesting = true
           done = false
-          if state["done"]
-            state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
-            done = true
-          else
-            begin
-              data = oai_get_records(
-                name,
-                content["repository"]["uri"],
-                format,
-                from,
-                to,
-                set,
-                resumption_token
-              )
-              state["resumption_token"] = nil
-              state["expiration_date_resumption_token"] = nil
-              if use_resumption_token
-                if data["resumptionToken"].is_a?(String)
-                  state["resumption_token"] = data["resumptionToken"]
-                elsif data["resumptionToken"].is_a?(Hash)
-                  data_token = data["resumptionToken"]
-                  state["resumption_token"] = data_token["$text"]
-                  state["expiration_date_resumption_token"] = data_token["_expirationDate"]
-                end
-              end
-              state["datetime_now"] = get_datetime_now
-              n_records = data["record"].size
-              state["count_harvested_records"] += n_records
-              if n_records > 0
-                timestamps = data["record"].map do |record|
-                  record["header"]["datestamp"]
-                end.sort
-                timestamp_latest = timestamps[-1].strftime('%FT%TZ')
-                state["latest_harvested_records_datestamp"] = convert_datetime(timestamp_latest, fmt_dt)
+          status = nil
+          begin
+            data = oai_get_records(
+              name,
+              content["repository"]["uri"],
+              format,
+              from,
+              to,
+              set,
+              resumption_token
+            )
+            state["resumption_token"] = nil
+            state["expiration_date_resumption_token"] = nil
+            if use_resumption_token
+              if data["resumptionToken"].is_a?(String)
+                state["resumption_token"] = data["resumptionToken"]
+              elsif data["resumptionToken"].is_a?(Hash)
+                data_token = data["resumptionToken"]
+                state["resumption_token"] = data_token["$text"]
+                state["expiration_date_resumption_token"] = data_token["_expirationDate"]
               end
-              state["count_success"] += 1
-            rescue StandardError => e
-              state["count_fails"] += 1
-              error = e
             end
-            if error.nil?
-              if use_resumption_token
-                if state["resumption_token"].nil?
-                  state_machine.add_event(EventHarvesting::DONE_HARVEST)
-                  to_pause = true
-                else
-                  to_pause = false
-                end
+            n_records = data["record"].size
+            state["count_harvested_records"] += n_records
+            if n_records > 0
+              timestamps = data["record"].map do |record|
+                record["header"]["datestamp"]
+              end.sort
+              timestamp_latest = timestamps[-1].strftime('%FT%TZ')
+              state["latest_harvested_records_datestamp"] = convert_datetime(timestamp_latest, fmt_dt)
+            end
+            state["count_success"] += 1
+          rescue StandardError => e
+            state["count_fails"] += 1
+            error = e
+          end
+          if error.nil?
+            err_info = nil
+            if use_resumption_token
+              if state["resumption_token"].nil?
+                state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
+                harvesting = false
+                status = "next harvesting: #{state["datetime_next_harvesting"]}"
               else
                 state_machine.add_event(EventHarvesting::DONE_HARVEST)
-                to_pause = true
+                harvesting = true
+                status = "harvesting"
               end
+            else
+              state_machine.add_event(EventHarvesting::DONE_HARVEST)
+              harvesting = true
+              status = "harvesting"
             end
-            state["done"] = done
+          else
+            err_info = {
+              "message" => error.message,
+              "backtrace" => error.backtrace
+            }
+            state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
+            harvesting = false
+            status = "error (see logs), next harvesting: #{state["datetime_next_harvesting"]}"
           end
+          state["done"] = done
+          state["harvesting"] = harvesting
+          state["datetime_now"] = get_datetime_now
+          state["status"] = status
+          state["error"] = err_info
           path_file_state = get_path_state_file_from_schedule_name(name)
           @logger.info("#{name}: writing to state file #{path_file_state}")
           write_state_file(path_file_state, state)
           @f_digest&.call(name, content, data, done, error, state, @logger)
-          break if to_pause
         when StateHarvesting::COMPLETE
           @logger.warn("#{name}: full harvesting complete")
-          content["active"] = false
-          handle_schedule_task(name)
+          state_machine.add_event(EventHarvesting::RESTART)
+          auto_deactivate_schedule = false
+          if auto_deactivate_schedule
+            content["active"] = false
+            handle_schedule_task(name)
+          end
           break
         else
           @logger.warn("#{name}: state #{state_machine.state} not known")
@@ -567,7 +604,18 @@ module OAISchedules
     end
 
     def get_datetime_now
-      DateTime.now.strftime('%FT%TZ')
+      Time.current.in_time_zone('Zulu').strftime('%FT%TZ')
+    end
+
+    def interval_iso8601_to_seconds(str_interval)
+      ISO8601::Duration.new(str_interval).to_seconds
+    end
+
+    def get_datetime_next_schedule_tick_from_now(str_interval)
+      interval_s = interval_iso8601_to_seconds(str_interval)
+      dt = Time.current.in_time_zone('Zulu')
+      dt += interval_s
+      dt.strftime('%FT%TZ')
     end
 
   end

data/lib/oai_schedules/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OaiSchedules
-  VERSION = "0.7.0"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,14 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: oai_schedules
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Davide Monari
 bindir: exe
 cert_chain: []
-date: 2025-07-09 00:00:00.000000000 Z
+date: 2025-07-15 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 8.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 8.0.2
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -78,12 +92,15 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- examples/dir_schedules/schedule_495607cf-f773-463f-8ee1-77d0f53e0c29.json
-- examples/dir_schedules/schedule_5e344861-806b-4361-98f7-a0be6a5984de.json
-- examples/dir_schedules/schedule_712f46ab-f87f-4db8-b69e-7101d9e2ae61.json
-- examples/dir_schedules/schedule_99922363-5b37-4438-a274-5a4a5167f811.json
-- examples/dir_schedules/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json
-- examples/dir_schedules/schedule_sample.json
+- examples/dir_schedules/.gitkeep
+- examples/dir_schedules_all/schedule_495607cf-f773-463f-8ee1-77d0f53e0c29.json
+- examples/dir_schedules_all/schedule_5e344861-806b-4361-98f7-a0be6a5984de.json
+- examples/dir_schedules_all/schedule_712f46ab-f87f-4db8-b69e-7101d9e2ae61.json
+- examples/dir_schedules_all/schedule_99922363-5b37-4438-a274-5a4a5167f811.json
+- examples/dir_schedules_all/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json
+- examples/dir_schedules_all/schedule_deventit.json
+- examples/dir_schedules_all/schedule_heron.json
+- examples/dir_schedules_all/schedule_sample.json
 - examples/dir_state/.gitkeep
 - examples/example_01.rb
 - examples/example_02.rb