oai_schedules 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b67e6f8703a12140e9f0ddb981a30a5311e80d27e5146bd9848601d27272db9
-  data.tar.gz: '09864239d4e38142c63a503a1535296f0a504bd4e9c0c84bfb87b024e640d104'
+  metadata.gz: 7d574b21314744d8ad56d1a1394d3fc778e860ba636d43569fedee7642ff3af2
+  data.tar.gz: b06822ea6110eedf0663f40f4c32120c321c0acd9a508deabe3204cb6f094962
 SHA512:
-  metadata.gz: cbe2109c9a0b04a3e743ef6c577a38320bc84d411bff334b9b5e5e6bfc536aa9946c536b4a4dd3f56f320977b734de16e183247ac64edbb8d5d0051c7690ae91
-  data.tar.gz: e747187b4375075a4e7f06831c67b6f59e7e2062578255732330884ea60bd4607b360054b2db504fcee7ec853d09b89ed8adf7f34d7f2f0a3d8ba49822a24b88
+  metadata.gz: 288bea5932749f50a8629e6fe7ebc041c68f19d39ca44a7513cbdfe1f24682f079f8d8434eabea6ce1cd1d1d3f40f71923ea4bcd701a5926589c544880de2b47
+  data.tar.gz: e2c309f3b419a605ad1eac62023bfdbb4102cd1ae7be8809659a42909851abcb90cfc22bccc790d21f5c43554f744f7bffaa40e006d7f67e91db77b20f50c0e6

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## [0.4.0] - 2025-03-28
+
+- Added logger to digestion function
+- Invalidated resumption token when either schedule set or format change
+- Extended error message on OAI URL fetch fail
+
+## [0.3.0] - 2025-03-26
+
+- Added schedule name and content to digestion function
+- Schedule content is deep-copied to internal state
+
 ## [0.2.0] - 2025-03-26
 
 - Added custom records digestions function

data/README.md

@@ -22,7 +22,7 @@ gem install oai_schedules
 ```ruby
 require 'oai_schedules/manager'
 
-f_show = lambda do |records, done|
+f_show = lambda do |name, content, records, done, logger|
   # ... do your stuff with records ...
   if done
     puts "done full harvesting"
@@ -58,7 +58,8 @@ At every iteration, every 2 seconds, it will get the *partial* records by using
 The code will do this by querying `https://eudml.org/oai/OAIHandler?verb=ListRecords&...` and adding the necessary
 query parameters.
 The custom function provided as `f_digest` will then be called at each iteration.
-This will be provided the partial list of `records` as a hash, and a `done` flag (full harvesting complete).
+This will be provided schedule `name` and `content`, the partial list of `records` as a hash, 
+a `done` flag (full harvesting complete), and the same logger used internally by the schedules manager.
 and it will write the new one to the state file, until no token is provided (end of the harvesting).
 As soon as the schedule is added, it is executed.
 It is possible to add all schedules in advance, then call `sleep` for infinite event loop.

data/examples/example_01.rb

@@ -4,7 +4,7 @@ require 'oai_schedules/manager'
 
 # usage with folder listener
 
-f_show = lambda do |records, done|
+f_show = lambda do |name, content, records, done, logger|
   # ... do your stuff with records ...
   if done
     puts "done full harvesting"

data/examples/example_02.rb

@@ -4,7 +4,7 @@ require 'oai_schedules/manager'
 
 # usage with programmatic schedules addition / modify / remove
 
-f_show = lambda do |records, done|
+f_show = lambda do |name, content, records, done, logger|
   # ... do your stuff with records ...
   if done
     puts "done full harvesting"

data/lib/oai_schedules/manager.rb

@@ -194,7 +194,7 @@ module OAISchedules
       task.add_observer(TaskObserver.new(@logger, name))
       # add item to schedules
       @schedules[name] = {
-        content: content,
+        content: deep_copy(content),
         task: task,
         state_machine: StateMachineHarvesting.new,
         state: state
@@ -213,7 +213,8 @@ module OAISchedules
 
 
     def modify_schedule(name, content)
-      @schedules[name][:content] = content
+      handle_schedule_state_at_schedule_change(name, content)
+      @schedules[name][:content] = deep_copy(content)
       handle_schedule_task(name)
     end
 
@@ -223,6 +224,22 @@ module OAISchedules
     private
 
 
+    def handle_schedule_state_at_schedule_change(name, content)
+      # invalidate resumption token if either schedule format or set changes
+      state = @schedules[name][:state]
+      if    (content["format"] != @schedules[name][:content]["format"]) \
+        ||  (content["set"] != @schedules[name][:content]["set"])
+        state["resumption_token"] = nil
+        state["count_partial_harversting"] = 0
+      end
+    end
+
+
+    def deep_copy(o)
+      Marshal.load(Marshal.dump(o))
+    end
+
+
     def handle_schedule_task(name)
       task = @schedules[name][:task]
       interval_s_safe = 60
@@ -360,7 +377,7 @@ module OAISchedules
             state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
             done = true
           end
-          @f_digest&.call(data, done)
+          @f_digest&.call(name, content, data, done, @logger)
           break
         when StateHarvesting::COMPLETE
           @logger.warn("#{name}: full harvesting complete")
@@ -412,7 +429,7 @@ module OAISchedules
       @logger.info("#{name}: fetching from #{url_query}")
       data = DataCollector::Core.filter(DataCollector::Input.new.from_uri(url_query), "$..#{verb}")
       if data.empty?
-        raise StandardError, "#{name}: GET #{url_query}: URL not available, or response has no element #{verb}"
+        raise StandardError, "#{name}: GET #{url_query}: URL not available, or response has no element #{verb}, or missing query parameters (e.g. set)"
       end
       data = data[0]
     rescue DataCollector::InputError => e

data/lib/oai_schedules/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OaiSchedules
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: oai_schedules
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Davide Monari
 bindir: exe
 cert_chain: []
-date: 2025-03-26 00:00:00.000000000 Z
+date: 2025-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby