oai_schedules 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1fe0345feb8fe60475f4668b9dfabadfe525b9251d60778cabcbb27f7c20d0b1
-  data.tar.gz: 4407a7a41d10b7a8dc30e8261a99eb74ea9cfdd5bc35e8592cbf27f76c114def
+  metadata.gz: 6b67e6f8703a12140e9f0ddb981a30a5311e80d27e5146bd9848601d27272db9
+  data.tar.gz: '09864239d4e38142c63a503a1535296f0a504bd4e9c0c84bfb87b024e640d104'
 SHA512:
-  metadata.gz: 29278455d3eceff2ce5b53bbb65f4074dd87b16a4389258fb77e4567b0d20343214d864ab166fcc8a7c2978ff2f8a20659946ee5fa990a15b1e1422f3d7bfcb1
-  data.tar.gz: 6d39fee175e2e2a8b872529ca79fac3b0c1b22f53f8502011e514bf1d769c99bacd37c066fb5682505a88c7d72284275876e52c66a5b2afa163674c4c0a0a42e
+  metadata.gz: cbe2109c9a0b04a3e743ef6c577a38320bc84d411bff334b9b5e5e6bfc536aa9946c536b4a4dd3f56f320977b734de16e183247ac64edbb8d5d0051c7690ae91
+  data.tar.gz: e747187b4375075a4e7f06831c67b6f59e7e2062578255732330884ea60bd4607b360054b2db504fcee7ec853d09b89ed8adf7f34d7f2f0a3d8ba49822a24b88

data/CHANGELOG.md

@@ -1,5 +1,7 @@
-## [Unreleased]
+## [0.2.0] - 2025-03-26
 
-## [0.1.0] - 2025-03-20
+- Added custom records digestions function
+
+## [0.1.0] - 2025-03-25
 
 - Initial release

data/README.md

@@ -1,28 +1,127 @@
-# OaiSchedules
+# Description
 
-TODO: Delete this and the text below, and describe your gem
-
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/oai_schedules`. To experiment with that code, run `bin/console` for an interactive prompt.
+This gem allows to create concurrent harvesting of records from OAI-PMH repositories, with custom-provided
+records digestion logic.
 
 ## Installation
 
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
-
 Install the gem and add to the application's Gemfile by executing:
 
 ```bash
-bundle add UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+bundle add oai_schedules
 ```
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
 ```bash
-gem install UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+gem install oai_schedules
 ```
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+require 'oai_schedules/manager'
+
+f_show = lambda do |records, done|
+  # ... do your stuff with records ...
+  if done
+    puts "done full harvesting"
+  end
+end
+
+manager = OAISchedules::Manager.new(
+  path_dir_state: "./dir_state",
+  f_digest: f_show
+)
+content_schedule = {
+  "interval" => "PT2S",
+  "active" => true,
+  "repository" => {
+    "uri" => "https://eudml.org/oai/OAIHandler"
+  },
+  "format" => "oai_dc",
+  "set" => "CEDRAM"
+}
+manager.add_schedule("my_sample_schedule", content_schedule)
+sleep
+```
+
+The demo app above instantiates a schedules manager.
+It will save schedules internal *state* file (e.g. OAI resumption token) in folder `./dir_state`.
+Each schedule state files is a JSON file in format *state_\<name-schedule\>.json*.
+The state file is used, in case of the schedules manager crash, to restore the harvesting from the last saved point.
+A schedule is then added to the manager. 
+The schedule with name `my_sample_schedule` will get partial records from set `CEDRAM` in `oai_dc` (Dublic Core) format,
+from the repository whose uri is `https://eudml.org/oai/OAIHandler`, every 2 seconds.
+At every iteration, every 2 seconds, it will get the *partial* records by using the previous OAI *resumption token*
+(for iterations after the very first one);
+The code will do this by querying `https://eudml.org/oai/OAIHandler?verb=ListRecords&...` and adding the necessary
+query parameters.
+The custom function provided as `f_digest` will then be called at each iteration.
+This will be provided the partial list of `records` as a hash, and a `done` flag (full harvesting complete).
+and it will write the new one to the state file, until no token is provided (end of the harvesting).
+As soon as the schedule is added, it is executed.
+It is possible to add all schedules in advance, then call `sleep` for infinite event loop.
+Ot it is also possible to do the following:
+
+```ruby
+# add schedule
+manager.add_schedule("my_sample_schedule", content_schedule)
+
+# ... do your own things here ...
+
+# modify schedule
+content_schedule["active"] = false  # e.g. pause schedule
+manager.modify_schedule("my_sample_schedule", content_schedule)
+
+# .... do other things ...
+
+# modify schedule
+content_schedule["active"] = true  # e.g. resume schedule ...
+content_schedule["interval"] = "PT5S" # ... but slower
+manager.modify_schedule("my_sample_schedule", content_schedule)
+
+# finally remove schedule
+manager.remove_schedule("my_sample_schedule")
+```
+
+It is also possible to listen to a folder containing *schedule* JSON files.
+These files must have this format: *schedule_\<name-schedule\>.json*.
+The manager will extract the schedule name from the file name.
+See the following:
+
+```ruby
+manager = OAISchedules::Manager.new(
+  # ...
+  path_dir_schedules: "./dir_schedules", 
+  # ...
+)
+# # alternative:
+# manager = OAISchedules::Manager.new()
+# manager.set_listener_dir_schedules("./dir_schedules")
+manager.run_listener_dir_schedules(block: true)
+```
+
+The above app will listen to files addition, modification and deletion in the folder `./dir_schedules`.
+The listener is started in blocking mode.
+
+You can add as many schedules are you want (with different names), they will all run concurrently.
+
+A *schedule definition*, either provided programmatically or from file, must have this structure:
+
+```json5
+{
+  "interval": "PT2S",     // (required) schedule interval in ISO8601 format.
+  "active": true,         // (required) is active (resumed) or not (paused)
+  "repository": {
+    "uri": "https://eudml.org/oai/OAIHandler"     // (required) OAI-PMH repository url
+  },
+  "format": "oai_dc",     // (required) metadata prefix to use
+  "set": "CEDRAM",        // (optional) set to collect. NOTE: on some repositories, this is necessary
+  "from": "2025-03-23T00:00:00Z",   // (optional) start datetime to collect from
+  "until": "2025-03-23T00:00:00Z",  // (optional) end datetime to collect from
+}
+```
 
 ## Development
 

data/examples/example_01.rb

@@ -4,8 +4,22 @@ require 'oai_schedules/manager'
 
 # usage with folder listener
 
-manager = OAISchedules::Manager.new(path_dir_schedules: "./dir_schedules", path_dir_state: "./dir_state")
+f_show = lambda do |records, done|
+  # ... do your stuff with records ...
+  if done
+    puts "done full harvesting"
+  end
+end
+
+manager = OAISchedules::Manager.new(
+  path_dir_schedules: "./dir_schedules",
+  path_dir_state: "./dir_state",
+  f_digest: f_show
+)
 # # alternative:
-# manager = OAISchedules::Manager.new()
+# manager = OAISchedules::Manager.new(
+#   path_dir_state: "./dir_state",
+#   f_digest: f_show
+# )
 # manager.set_listener_dir_schedules("./dir_schedules")
 manager.run_listener_dir_schedules(block: true)

data/examples/example_02.rb

@@ -4,14 +4,25 @@ require 'oai_schedules/manager'
 
 # usage with programmatic schedules addition / modify / remove
 
-manager = OAISchedules::Manager.new(path_dir_state: "./dir_state")
+f_show = lambda do |records, done|
+  # ... do your stuff with records ...
+  if done
+    puts "done full harvesting"
+  end
+end
+
+manager = OAISchedules::Manager.new(
+  path_dir_state: "./dir_state",
+  f_digest: f_show
+)
 content_schedule = {
   "interval" => "PT2S",
   "active" => true,
   "repository" => {
     "uri" => "https://eudml.org/oai/OAIHandler"
   },
-  "format" => "oai_dc"
+  "format" => "oai_dc",
+  "set" => "CEDRAM"
 }
 manager.add_schedule("my_sample_schedule", content_schedule)
 sleep

data/lib/oai_schedules/manager.rb

@@ -119,7 +119,7 @@ module OAISchedules
 
     attr_accessor :path_dir_state
 
-    def initialize(path_dir_schedules: nil, path_dir_state: nil)
+    def initialize(path_dir_schedules: nil, path_dir_state: nil, f_digest: nil)
       @logger = Logger.new(STDOUT)
       @path_dir_state = path_dir_state
       @schedules = {}
@@ -128,6 +128,7 @@ module OAISchedules
       unless path_dir_schedules.nil?
         set_listener_dir_schedules(path_dir_schedules)
       end
+      @f_digest = f_digest
     end
 
 
@@ -354,9 +355,12 @@ module OAISchedules
           write_state_file(path_file_state, state)
           if !data["resumptionToken"].nil?
             state_machine.add_event(EventHarvesting::DONE_HARVEST)
+            done = false
           else
             state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
+            done = true
           end
+          @f_digest&.call(data, done)
           break
         when StateHarvesting::COMPLETE
           @logger.warn("#{name}: full harvesting complete")

data/lib/oai_schedules/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OaiSchedules
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: oai_schedules
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Davide Monari
 bindir: exe
 cert_chain: []
-date: 2025-03-25 00:00:00.000000000 Z
+date: 2025-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -65,7 +65,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.9.0
-description: gem to run concurrent OAI-PHM harvesting schedules
+description: gem to run concurrent OAI-PMH harvesting schedules
 email:
 - davide.monari@kuleuven.be
 executables: []
@@ -110,5 +110,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 3.6.2
 specification_version: 4
-summary: gem to run concurrent OAI-PHM harvesting schedules
+summary: gem to run concurrent OAI-PMH harvesting schedules
 test_files: []