oai_schedules 0.1.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +41 -2
- data/README.md +115 -9
- data/examples/dir_schedules/.gitkeep +0 -0
- data/examples/dir_schedules_all/schedule_495607cf-f773-463f-8ee1-77d0f53e0c29.json +38 -0
- data/examples/dir_schedules_all/schedule_5e344861-806b-4361-98f7-a0be6a5984de.json +38 -0
- data/examples/dir_schedules_all/schedule_712f46ab-f87f-4db8-b69e-7101d9e2ae61.json +41 -0
- data/examples/dir_schedules_all/schedule_99922363-5b37-4438-a274-5a4a5167f811.json +62 -0
- data/examples/dir_schedules_all/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json +62 -0
- data/examples/dir_schedules_all/schedule_deventit.json +41 -0
- data/examples/dir_schedules_all/schedule_heron.json +38 -0
- data/examples/{dir_schedules → dir_schedules_all}/schedule_sample.json +4 -3
- data/examples/example_01.rb +22 -2
- data/examples/example_02.rb +19 -2
- data/lib/oai_schedules/manager.rb +221 -46
- data/lib/oai_schedules/version.rb +1 -1
- metadata +41 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 62fc2527845220bd45d6c4dde26efb5711088c40bd84d9d0b55dfe17847709b0
|
|
4
|
+
data.tar.gz: ac69ccc58b501431a09e11aee1051bd7c76987fe9b53a001f4a3dbed2da278e1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5df36bbaa9f8b7e6e9f0a90da604e2f2053a5927ce97ffdaa019ed24af6f01b6d23277b536509a317886f9371d632ab5cf37ff1bb54aa84441ccb99529970169
|
|
7
|
+
data.tar.gz: e206211d4ee16a8428edb15f408f4ac6a1bcefbdef2998546d2e66ecc4e792954e30a0e132e37fdc00d5813394fa844637d073deff271240a649051f3c8098c9
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,44 @@
|
|
|
1
|
-
## [
|
|
1
|
+
## [0.9.1] - 2025-07-15
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
- Added required gem
|
|
4
|
+
|
|
5
|
+
## [0.9.0] - 2025-07-15
|
|
6
|
+
|
|
7
|
+
- Restored resumption token logic (the only one correct)
|
|
8
|
+
- Fixed Zulu time issues
|
|
9
|
+
- Added more info in the schedule state
|
|
10
|
+
|
|
11
|
+
## [0.8.0] - 2025-07-10
|
|
12
|
+
|
|
13
|
+
- Complete harvesting is now within a single thread tick
|
|
14
|
+
- Added more info in the schedule state
|
|
15
|
+
|
|
16
|
+
## [0.7.0] - 2025-07-09
|
|
17
|
+
|
|
18
|
+
- Partial harvest policy changed: avoid using resumption tokens
|
|
19
|
+
(adds complication due to expiration time), and change "from" query param
|
|
20
|
+
as datestamp of the most recent record fetched + eps (from granularity)
|
|
21
|
+
- Bugfix: "from" and "to" query params now have correct format based on granularity
|
|
22
|
+
- Better exceptions management
|
|
23
|
+
- Some cleanup and sparse bugfixes
|
|
24
|
+
- Added more sample schedules
|
|
25
|
+
- Added more info in the schedule state
|
|
26
|
+
|
|
27
|
+
## [0.4.0] - 2025-03-28
|
|
28
|
+
|
|
29
|
+
- Added logger to digestion function
|
|
30
|
+
- Invalidated resumption token when either schedule set or format change
|
|
31
|
+
- Extended error message on OAI URL fetch fail
|
|
32
|
+
|
|
33
|
+
## [0.3.0] - 2025-03-26
|
|
34
|
+
|
|
35
|
+
- Added schedule name and content to digestion function
|
|
36
|
+
- Schedule content is deep-copied to internal state
|
|
37
|
+
|
|
38
|
+
## [0.2.0] - 2025-03-26
|
|
39
|
+
|
|
40
|
+
- Added custom records digestions function
|
|
41
|
+
|
|
42
|
+
## [0.1.0] - 2025-03-25
|
|
4
43
|
|
|
5
44
|
- Initial release
|
data/README.md
CHANGED
|
@@ -1,28 +1,134 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Description
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/oai_schedules`. To experiment with that code, run `bin/console` for an interactive prompt.
|
|
3
|
+
This gem allows to create concurrent harvesting of records from OAI-PMH repositories, with custom-provided
|
|
4
|
+
records digestion logic.
|
|
6
5
|
|
|
7
6
|
## Installation
|
|
8
7
|
|
|
9
|
-
TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
|
|
10
|
-
|
|
11
8
|
Install the gem and add to the application's Gemfile by executing:
|
|
12
9
|
|
|
13
10
|
```bash
|
|
14
|
-
bundle add
|
|
11
|
+
bundle add oai_schedules
|
|
15
12
|
```
|
|
16
13
|
|
|
17
14
|
If bundler is not being used to manage dependencies, install the gem by executing:
|
|
18
15
|
|
|
19
16
|
```bash
|
|
20
|
-
gem install
|
|
17
|
+
gem install oai_schedules
|
|
21
18
|
```
|
|
22
19
|
|
|
23
20
|
## Usage
|
|
24
21
|
|
|
25
|
-
|
|
22
|
+
```ruby
|
|
23
|
+
require 'oai_schedules/manager'
|
|
24
|
+
|
|
25
|
+
f_show = lambda do |name, content, records, done, error, state, logger|
|
|
26
|
+
if error.nil?
|
|
27
|
+
# ... do your stuff with records ...
|
|
28
|
+
else
|
|
29
|
+
puts error.message
|
|
30
|
+
end
|
|
31
|
+
puts state
|
|
32
|
+
if done
|
|
33
|
+
puts "done full harvesting"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
manager = OAISchedules::Manager.new(
|
|
38
|
+
path_dir_state: "./dir_state",
|
|
39
|
+
f_digest: f_show
|
|
40
|
+
)
|
|
41
|
+
content_schedule = {
|
|
42
|
+
"interval" => "PT2S",
|
|
43
|
+
"active" => true,
|
|
44
|
+
"repository" => {
|
|
45
|
+
"uri" => "https://eudml.org/oai/OAIHandler"
|
|
46
|
+
},
|
|
47
|
+
"format" => "oai_dc",
|
|
48
|
+
"set" => "CEDRAM"
|
|
49
|
+
}
|
|
50
|
+
manager.add_schedule("my_sample_schedule", content_schedule)
|
|
51
|
+
sleep
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The demo app above instantiates a schedules manager.
|
|
55
|
+
It will save schedules internal *state* file (e.g. OAI resumption token) in folder `./dir_state`.
|
|
56
|
+
Each schedule state files is a JSON file in format *state_\<name-schedule\>.json*.
|
|
57
|
+
The state file is used, in case of the schedules manager crash, to restore the harvesting from the last saved point.
|
|
58
|
+
A schedule is then added to the manager.
|
|
59
|
+
The schedule with name `my_sample_schedule` will get partial records from set `CEDRAM` in `oai_dc` (Dublic Core) format,
|
|
60
|
+
from the repository whose uri is `https://eudml.org/oai/OAIHandler`, every 2 seconds.
|
|
61
|
+
At every iteration, every 2 seconds, it will get the *partial* records by using the previous OAI *resumption token*
|
|
62
|
+
(for iterations after the very first one);
|
|
63
|
+
The code will do this by querying `https://eudml.org/oai/OAIHandler?verb=ListRecords&...` and adding the necessary
|
|
64
|
+
query parameters.
|
|
65
|
+
The custom function provided as `f_digest` will then be called at each iteration.
|
|
66
|
+
This will be provided schedule `name` and `content`, the partial list of `records` as a hash,
|
|
67
|
+
a `done` flag (full harvesting complete), an `error` exception if happened, the harvesting `state`,
|
|
68
|
+
and the same logger used internally by the schedules manager.
|
|
69
|
+
and it will write the new one to the state file, until no token is provided (end of the harvesting).
|
|
70
|
+
As soon as the schedule is added, it is executed.
|
|
71
|
+
It is possible to add all schedules in advance, then call `sleep` for infinite event loop.
|
|
72
|
+
Ot it is also possible to do the following:
|
|
73
|
+
|
|
74
|
+
```ruby
|
|
75
|
+
# add schedule
|
|
76
|
+
manager.add_schedule("my_sample_schedule", content_schedule)
|
|
77
|
+
|
|
78
|
+
# ... do your own things here ...
|
|
79
|
+
|
|
80
|
+
# modify schedule
|
|
81
|
+
content_schedule["active"] = false # e.g. pause schedule
|
|
82
|
+
manager.modify_schedule("my_sample_schedule", content_schedule)
|
|
83
|
+
|
|
84
|
+
# .... do other things ...
|
|
85
|
+
|
|
86
|
+
# modify schedule
|
|
87
|
+
content_schedule["active"] = true # e.g. resume schedule ...
|
|
88
|
+
content_schedule["interval"] = "PT5S" # ... but slower
|
|
89
|
+
manager.modify_schedule("my_sample_schedule", content_schedule)
|
|
90
|
+
|
|
91
|
+
# finally remove schedule
|
|
92
|
+
manager.remove_schedule("my_sample_schedule")
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
It is also possible to listen to a folder containing *schedule* JSON files.
|
|
96
|
+
These files must have this format: *schedule_\<name-schedule\>.json*.
|
|
97
|
+
The manager will extract the schedule name from the file name.
|
|
98
|
+
See the following:
|
|
99
|
+
|
|
100
|
+
```ruby
|
|
101
|
+
manager = OAISchedules::Manager.new(
|
|
102
|
+
# ...
|
|
103
|
+
path_dir_schedules: "./dir_schedules",
|
|
104
|
+
# ...
|
|
105
|
+
)
|
|
106
|
+
# # alternative:
|
|
107
|
+
# manager = OAISchedules::Manager.new()
|
|
108
|
+
# manager.set_listener_dir_schedules("./dir_schedules")
|
|
109
|
+
manager.run_listener_dir_schedules(block: true)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The above app will listen to files addition, modification and deletion in the folder `./dir_schedules`.
|
|
113
|
+
The listener is started in blocking mode.
|
|
114
|
+
|
|
115
|
+
You can add as many schedules are you want (with different names), they will all run concurrently.
|
|
116
|
+
|
|
117
|
+
A *schedule definition*, either provided programmatically or from file, must have this structure:
|
|
118
|
+
|
|
119
|
+
```json5
|
|
120
|
+
{
|
|
121
|
+
"interval": "PT2S", // (required) schedule interval in ISO8601 format.
|
|
122
|
+
"active": true, // (required) is active (resumed) or not (paused)
|
|
123
|
+
"repository": {
|
|
124
|
+
"uri": "https://eudml.org/oai/OAIHandler" // (required) OAI-PMH repository url
|
|
125
|
+
},
|
|
126
|
+
"format": "oai_dc", // (required) metadata prefix to use
|
|
127
|
+
"set": "CEDRAM", // (optional) set to collect. NOTE: on some repositories, this is necessary
|
|
128
|
+
"from": "2025-03-23T00:00:00Z", // (optional) start datetime to collect from
|
|
129
|
+
"until": "2025-03-23T00:00:00Z", // (optional) end datetime to collect from
|
|
130
|
+
}
|
|
131
|
+
```
|
|
26
132
|
|
|
27
133
|
## Development
|
|
28
134
|
|
|
File without changes
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"interval": "PT5S",
|
|
3
|
+
"repository": {
|
|
4
|
+
"uri": "https://opac.amsab.be/OAI/Server",
|
|
5
|
+
"repository_name": "Amsab Online Catalog",
|
|
6
|
+
"protocol_version": "2.0",
|
|
7
|
+
"admin_email": [
|
|
8
|
+
"help@amsab.be"
|
|
9
|
+
],
|
|
10
|
+
"earliest_datestamp": "2000-01-01T00:00:00+00:00",
|
|
11
|
+
"deleted_records": "transient",
|
|
12
|
+
"granularity": "YYYY-MM-DDThh:mm:ssZ",
|
|
13
|
+
"metadata_format": [
|
|
14
|
+
"oai_dc",
|
|
15
|
+
"marc21"
|
|
16
|
+
],
|
|
17
|
+
"set": [
|
|
18
|
+
"collection"
|
|
19
|
+
],
|
|
20
|
+
"id": "490cac7921d1801e6d398aa09bca3ef8f51d07e7dae7cc25676d1ea94126483f"
|
|
21
|
+
},
|
|
22
|
+
"active": true,
|
|
23
|
+
"transformer": {
|
|
24
|
+
"transformer_name": [
|
|
25
|
+
"dummy_transformer"
|
|
26
|
+
],
|
|
27
|
+
"type": {
|
|
28
|
+
"id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
|
|
29
|
+
"value": "dummy_type"
|
|
30
|
+
},
|
|
31
|
+
"uri": "http://dummy-uri.org/",
|
|
32
|
+
"id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
|
|
33
|
+
},
|
|
34
|
+
"format": "oai_dc",
|
|
35
|
+
"set": "collection",
|
|
36
|
+
"from": "2000-01-01T00:00:00+00:00",
|
|
37
|
+
"id": "495607cf-f773-463f-8ee1-77d0f53e0c29"
|
|
38
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"interval": "PT5S",
|
|
3
|
+
"repository": {
|
|
4
|
+
"uri": "https://heron.libis.be/ca_veb_q/admin/service.php/OAI/cw_organisaties/request",
|
|
5
|
+
"repository_name": "VEB",
|
|
6
|
+
"protocol_version": "2.0",
|
|
7
|
+
"admin_email": [
|
|
8
|
+
"collectiveaccess@vlaamse-erfgoedbibliotheken.be"
|
|
9
|
+
],
|
|
10
|
+
"earliest_datestamp": "2020-08-17T12:55:02+00:00",
|
|
11
|
+
"deleted_records": "transient",
|
|
12
|
+
"granularity": "YYYY-MM-DDThh:mm:ssZ",
|
|
13
|
+
"metadata_format": [
|
|
14
|
+
"oai_dc",
|
|
15
|
+
"oai_veb"
|
|
16
|
+
],
|
|
17
|
+
"set": [
|
|
18
|
+
"1"
|
|
19
|
+
],
|
|
20
|
+
"id": "4886e3793003d2b1dffbbdb41ca13024c7c809892314ab301ded0d7ecfd7b469"
|
|
21
|
+
},
|
|
22
|
+
"active": true,
|
|
23
|
+
"transformer": {
|
|
24
|
+
"transformer_name": [
|
|
25
|
+
"dummy_transformer"
|
|
26
|
+
],
|
|
27
|
+
"type": {
|
|
28
|
+
"id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
|
|
29
|
+
"value": "dummy_type"
|
|
30
|
+
},
|
|
31
|
+
"uri": "http://dummy-uri.org/",
|
|
32
|
+
"id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
|
|
33
|
+
},
|
|
34
|
+
"format": "oai_dc",
|
|
35
|
+
"set": "1",
|
|
36
|
+
"from": "2025-06-10T12:39:00+00:00",
|
|
37
|
+
"id": "5e344861-806b-4361-98f7-a0be6a5984de"
|
|
38
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"interval": "PT5S",
|
|
3
|
+
"repository": {
|
|
4
|
+
"uri": "http://lag.hosting.deventit.net/atlantispubliek/oai.axd",
|
|
5
|
+
"repository_name": "Liberas",
|
|
6
|
+
"protocol_version": "2.0",
|
|
7
|
+
"admin_email": [
|
|
8
|
+
"support@deventit.nl"
|
|
9
|
+
],
|
|
10
|
+
"earliest_datestamp": "1900-01-01T01:01:01+00:00",
|
|
11
|
+
"deleted_records": "persistent",
|
|
12
|
+
"granularity": "YYYY-MM-DDThh:mm:ssZ",
|
|
13
|
+
"metadata_format": [
|
|
14
|
+
"APEX",
|
|
15
|
+
"EAC",
|
|
16
|
+
"EAD"
|
|
17
|
+
],
|
|
18
|
+
"set": [
|
|
19
|
+
"APEX",
|
|
20
|
+
"APEXAO",
|
|
21
|
+
"EAC",
|
|
22
|
+
"EAD"
|
|
23
|
+
],
|
|
24
|
+
"id": "93b6d240ac9b782664f18823b761a128fc19116506dd228763b52ba8fb64e1b9"
|
|
25
|
+
},
|
|
26
|
+
"active": true,
|
|
27
|
+
"transformer": {
|
|
28
|
+
"transformer_name": [
|
|
29
|
+
"dummy_transformer"
|
|
30
|
+
],
|
|
31
|
+
"type": {
|
|
32
|
+
"id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
|
|
33
|
+
"value": "dummy_type"
|
|
34
|
+
},
|
|
35
|
+
"uri": "http://dummy-uri.org/",
|
|
36
|
+
"id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
|
|
37
|
+
},
|
|
38
|
+
"format": "EAD",
|
|
39
|
+
"set": "EAD",
|
|
40
|
+
"id": "712f46ab-f87f-4db8-b69e-7101d9e2ae61"
|
|
41
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
{
|
|
2
|
+
"interval": "PT5S",
|
|
3
|
+
"repository": {
|
|
4
|
+
"uri": "https://eudml.org/oai/OAIHandler",
|
|
5
|
+
"repository_name": "REPOX Repository",
|
|
6
|
+
"protocol_version": "2.0",
|
|
7
|
+
"admin_email": [
|
|
8
|
+
"mailto:gilberto.pedrosa@ist.utl.pt"
|
|
9
|
+
],
|
|
10
|
+
"earliest_datestamp": "1970-01-01T00:00:00+00:00",
|
|
11
|
+
"deleted_records": "persistent",
|
|
12
|
+
"granularity": "YYYY-MM-DD",
|
|
13
|
+
"metadata_format": [
|
|
14
|
+
"oai_dc",
|
|
15
|
+
"ese",
|
|
16
|
+
"eudml-article2",
|
|
17
|
+
"eudml-book2"
|
|
18
|
+
],
|
|
19
|
+
"set": [
|
|
20
|
+
"BDIM",
|
|
21
|
+
"BulDML",
|
|
22
|
+
"CEDRAM",
|
|
23
|
+
"DMLE",
|
|
24
|
+
"DML_CZ_Monograph",
|
|
25
|
+
"DML_CZ_Proceeding",
|
|
26
|
+
"DML_CZ_Serial",
|
|
27
|
+
"EDPS",
|
|
28
|
+
"ELibM",
|
|
29
|
+
"GALLICA",
|
|
30
|
+
"GDZ_Band",
|
|
31
|
+
"GDZ_Mathematica",
|
|
32
|
+
"GDZ_Monographs",
|
|
33
|
+
"GDZ_RusDML",
|
|
34
|
+
"HDML_Books",
|
|
35
|
+
"HDML_Conferences",
|
|
36
|
+
"HDML_Journals",
|
|
37
|
+
"MISANU",
|
|
38
|
+
"NUMDAM",
|
|
39
|
+
"NUMDAM_book",
|
|
40
|
+
"PLDML",
|
|
41
|
+
"PLDML_book",
|
|
42
|
+
"PMath"
|
|
43
|
+
],
|
|
44
|
+
"id": "02df523af427deb93b7cb4600ca347f9297d0e31d51c2783c634459dac457bd0"
|
|
45
|
+
},
|
|
46
|
+
"active": true,
|
|
47
|
+
"transformer": {
|
|
48
|
+
"transformer_name": [
|
|
49
|
+
"dummy_transformer"
|
|
50
|
+
],
|
|
51
|
+
"type": {
|
|
52
|
+
"id": "b1671aad-e825-4b5a-b50b-d8591b425e2a",
|
|
53
|
+
"value": "dummy_type"
|
|
54
|
+
},
|
|
55
|
+
"uri": "http://dummy-uri.org/",
|
|
56
|
+
"id": "7889ce03-28d9-479d-bb9a-b239f179453a"
|
|
57
|
+
},
|
|
58
|
+
"format": "oai_dc",
|
|
59
|
+
"set": "CEDRAM",
|
|
60
|
+
"from": "1970-01-01T00:00:00+00:00",
|
|
61
|
+
"id": "99922363-5b37-4438-a274-5a4a5167f811"
|
|
62
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
{
|
|
2
|
+
"interval": "PT5S",
|
|
3
|
+
"repository": {
|
|
4
|
+
"uri": "https://eudml.org/oai/OAIHandler",
|
|
5
|
+
"repository_name": "REPOX Repository",
|
|
6
|
+
"protocol_version": "2.0",
|
|
7
|
+
"admin_email": [
|
|
8
|
+
"mailto:gilberto.pedrosa@ist.utl.pt"
|
|
9
|
+
],
|
|
10
|
+
"earliest_datestamp": "1970-01-01T00:00:00+00:00",
|
|
11
|
+
"deleted_records": "persistent",
|
|
12
|
+
"granularity": "YYYY-MM-DD",
|
|
13
|
+
"metadata_format": [
|
|
14
|
+
"oai_dc",
|
|
15
|
+
"ese",
|
|
16
|
+
"eudml-article2",
|
|
17
|
+
"eudml-book2"
|
|
18
|
+
],
|
|
19
|
+
"set": [
|
|
20
|
+
"BDIM",
|
|
21
|
+
"BulDML",
|
|
22
|
+
"CEDRAM",
|
|
23
|
+
"DMLE",
|
|
24
|
+
"DML_CZ_Monograph",
|
|
25
|
+
"DML_CZ_Proceeding",
|
|
26
|
+
"DML_CZ_Serial",
|
|
27
|
+
"EDPS",
|
|
28
|
+
"ELibM",
|
|
29
|
+
"GALLICA",
|
|
30
|
+
"GDZ_Band",
|
|
31
|
+
"GDZ_Mathematica",
|
|
32
|
+
"GDZ_Monographs",
|
|
33
|
+
"GDZ_RusDML",
|
|
34
|
+
"HDML_Books",
|
|
35
|
+
"HDML_Conferences",
|
|
36
|
+
"HDML_Journals",
|
|
37
|
+
"MISANU",
|
|
38
|
+
"NUMDAM",
|
|
39
|
+
"NUMDAM_book",
|
|
40
|
+
"PLDML",
|
|
41
|
+
"PLDML_book",
|
|
42
|
+
"PMath"
|
|
43
|
+
],
|
|
44
|
+
"id": "02df523af427deb93b7cb4600ca347f9297d0e31d51c2783c634459dac457bd0"
|
|
45
|
+
},
|
|
46
|
+
"active": true,
|
|
47
|
+
"transformer": {
|
|
48
|
+
"transformer_name": [
|
|
49
|
+
"dummy_transformer"
|
|
50
|
+
],
|
|
51
|
+
"type": {
|
|
52
|
+
"id": "b1671aad-e825-4b5a-b50b-d8591b425e2a",
|
|
53
|
+
"value": "dummy_type"
|
|
54
|
+
},
|
|
55
|
+
"uri": "http://dummy-uri.org/",
|
|
56
|
+
"id": "7889ce03-28d9-479d-bb9a-b239f179453a"
|
|
57
|
+
},
|
|
58
|
+
"format": "oai_dc",
|
|
59
|
+
"set": "CEDRAM",
|
|
60
|
+
"from": "1970-01-01T00:00:00+00:00",
|
|
61
|
+
"id": "dc34623d-2ae2-4e90-91d8-26f4ba29a056"
|
|
62
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"interval": "PT5S",
|
|
3
|
+
"repository": {
|
|
4
|
+
"uri": "http://lag.hosting.deventit.net/atlantispubliek/oai.axd",
|
|
5
|
+
"repository_name": "Liberas",
|
|
6
|
+
"protocol_version": "2.0",
|
|
7
|
+
"admin_email": [
|
|
8
|
+
"support@deventit.nl"
|
|
9
|
+
],
|
|
10
|
+
"earliest_datestamp": "1900-01-01T01:01:01+00:00",
|
|
11
|
+
"deleted_records": "persistent",
|
|
12
|
+
"granularity": "YYYY-MM-DDThh:mm:ssZ",
|
|
13
|
+
"metadata_format": [
|
|
14
|
+
"APEX",
|
|
15
|
+
"EAC",
|
|
16
|
+
"EAD"
|
|
17
|
+
],
|
|
18
|
+
"set": [
|
|
19
|
+
"APEX",
|
|
20
|
+
"APEXAO",
|
|
21
|
+
"EAC",
|
|
22
|
+
"EAD"
|
|
23
|
+
],
|
|
24
|
+
"id": "93b6d240ac9b782664f18823b761a128fc19116506dd228763b52ba8fb64e1b9"
|
|
25
|
+
},
|
|
26
|
+
"active": true,
|
|
27
|
+
"transformer": {
|
|
28
|
+
"transformer_name": [
|
|
29
|
+
"dummy_transformer"
|
|
30
|
+
],
|
|
31
|
+
"type": {
|
|
32
|
+
"id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
|
|
33
|
+
"value": "dummy_type"
|
|
34
|
+
},
|
|
35
|
+
"uri": "http://dummy-uri.org/",
|
|
36
|
+
"id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
|
|
37
|
+
},
|
|
38
|
+
"format": "EAD",
|
|
39
|
+
"set": "EAD",
|
|
40
|
+
"id": "deventit"
|
|
41
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"interval": "PT5S",
|
|
3
|
+
"repository": {
|
|
4
|
+
"uri": "https://heron.libis.be/ca_veb_q/admin/service.php/OAI/cw_organisaties/request",
|
|
5
|
+
"repository_name": "VEB",
|
|
6
|
+
"protocol_version": "2.0",
|
|
7
|
+
"admin_email": [
|
|
8
|
+
"collectiveaccess@vlaamse-erfgoedbibliotheken.be"
|
|
9
|
+
],
|
|
10
|
+
"earliest_datestamp": "2020-08-17T12:55:02+00:00",
|
|
11
|
+
"deleted_records": "transient",
|
|
12
|
+
"granularity": "YYYY-MM-DDThh:mm:ssZ",
|
|
13
|
+
"metadata_format": [
|
|
14
|
+
"oai_dc",
|
|
15
|
+
"oai_veb"
|
|
16
|
+
],
|
|
17
|
+
"set": [
|
|
18
|
+
"1"
|
|
19
|
+
],
|
|
20
|
+
"id": "4886e3793003d2b1dffbbdb41ca13024c7c809892314ab301ded0d7ecfd7b469"
|
|
21
|
+
},
|
|
22
|
+
"active": true,
|
|
23
|
+
"transformer": {
|
|
24
|
+
"transformer_name": [
|
|
25
|
+
"dummy_transformer"
|
|
26
|
+
],
|
|
27
|
+
"type": {
|
|
28
|
+
"id": "bb8bfd5d-d914-41ae-b9f1-b5065299d9b9",
|
|
29
|
+
"value": "dummy_type"
|
|
30
|
+
},
|
|
31
|
+
"uri": "http://dummy-uri.org/",
|
|
32
|
+
"id": "dd377ec0-c890-4635-b6d8-553ddf609c01"
|
|
33
|
+
},
|
|
34
|
+
"format": "oai_veb",
|
|
35
|
+
"set": "1",
|
|
36
|
+
"from": "1900-06-10T12:39:00+00:00",
|
|
37
|
+
"id": "heron"
|
|
38
|
+
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
{
|
|
2
|
-
"interval": "
|
|
2
|
+
"interval": "PT5S",
|
|
3
3
|
"active": true,
|
|
4
4
|
"repository": {
|
|
5
|
-
"uri": "https://eu.alma.exlibrisgroup.com/view/oai/32KUL_KUL/request"
|
|
5
|
+
"uri": "https://eu.alma.exlibrisgroup.com/view/oai/32KUL_KUL/request",
|
|
6
|
+
"granularity": "YYYY-MM-DDThh:mm:ssZ"
|
|
6
7
|
},
|
|
7
8
|
"format": "marc21",
|
|
8
9
|
"set": "KUL_Rapid_Journals_Print_LendableInternational",
|
|
9
|
-
"from": "
|
|
10
|
+
"from": "2024-03-01T00:00:00Z"
|
|
10
11
|
}
|
data/examples/example_01.rb
CHANGED
|
@@ -4,8 +4,28 @@ require 'oai_schedules/manager'
|
|
|
4
4
|
|
|
5
5
|
# usage with folder listener
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
f_show = lambda do |name, content, records, done, error, state, logger|
|
|
8
|
+
if error.nil?
|
|
9
|
+
# ... do your stuff with records ...
|
|
10
|
+
else
|
|
11
|
+
puts error.message
|
|
12
|
+
puts error.backtrace
|
|
13
|
+
end
|
|
14
|
+
puts JSON.pretty_generate(state)
|
|
15
|
+
if done
|
|
16
|
+
puts "done full harvesting"
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
manager = OAISchedules::Manager.new(
|
|
21
|
+
path_dir_schedules: "./dir_schedules",
|
|
22
|
+
path_dir_state: "./dir_state",
|
|
23
|
+
f_digest: f_show
|
|
24
|
+
)
|
|
8
25
|
# # alternative:
|
|
9
|
-
# manager = OAISchedules::Manager.new(
|
|
26
|
+
# manager = OAISchedules::Manager.new(
|
|
27
|
+
# path_dir_state: "./dir_state",
|
|
28
|
+
# f_digest: f_show
|
|
29
|
+
# )
|
|
10
30
|
# manager.set_listener_dir_schedules("./dir_schedules")
|
|
11
31
|
manager.run_listener_dir_schedules(block: true)
|
data/examples/example_02.rb
CHANGED
|
@@ -4,14 +4,31 @@ require 'oai_schedules/manager'
|
|
|
4
4
|
|
|
5
5
|
# usage with programmatic schedules addition / modify / remove
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
f_show = lambda do |name, content, records, done, error, state, logger|
|
|
8
|
+
if error.nil?
|
|
9
|
+
# ... do your stuff with records ...
|
|
10
|
+
else
|
|
11
|
+
puts error.message
|
|
12
|
+
puts error.backtrace
|
|
13
|
+
end
|
|
14
|
+
puts JSON.pretty_generate(state)
|
|
15
|
+
if done
|
|
16
|
+
puts "done full harvesting"
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
manager = OAISchedules::Manager.new(
|
|
21
|
+
path_dir_state: "./dir_state",
|
|
22
|
+
f_digest: f_show
|
|
23
|
+
)
|
|
8
24
|
content_schedule = {
|
|
9
25
|
"interval" => "PT2S",
|
|
10
26
|
"active" => true,
|
|
11
27
|
"repository" => {
|
|
12
28
|
"uri" => "https://eudml.org/oai/OAIHandler"
|
|
13
29
|
},
|
|
14
|
-
"format" => "oai_dc"
|
|
30
|
+
"format" => "oai_dc",
|
|
31
|
+
"set" => "CEDRAM"
|
|
15
32
|
}
|
|
16
33
|
manager.add_schedule("my_sample_schedule", content_schedule)
|
|
17
34
|
sleep
|
|
@@ -7,7 +7,8 @@ require 'concurrent-ruby'
|
|
|
7
7
|
require 'logger'
|
|
8
8
|
require 'data_collector'
|
|
9
9
|
require 'iso8601'
|
|
10
|
-
|
|
10
|
+
require 'date'
|
|
11
|
+
require 'active_support/core_ext/time'
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
|
|
@@ -49,6 +50,7 @@ module OAISchedules
|
|
|
49
50
|
REQUEST_HARVEST = 2
|
|
50
51
|
DONE_HARVEST = 3
|
|
51
52
|
DONE_FULL_HARVEST = 4
|
|
53
|
+
RESTART = 5
|
|
52
54
|
end
|
|
53
55
|
|
|
54
56
|
class StateMachineHarvesting
|
|
@@ -103,6 +105,15 @@ module OAISchedules
|
|
|
103
105
|
@state
|
|
104
106
|
end
|
|
105
107
|
|
|
108
|
+
when StateHarvesting::COMPLETE
|
|
109
|
+
|
|
110
|
+
case event
|
|
111
|
+
when EventHarvesting::RESTART
|
|
112
|
+
@state = StateHarvesting::NOT_IDENTIFIED
|
|
113
|
+
else
|
|
114
|
+
@state
|
|
115
|
+
end
|
|
116
|
+
|
|
106
117
|
else
|
|
107
118
|
@state
|
|
108
119
|
end
|
|
@@ -119,7 +130,7 @@ module OAISchedules
|
|
|
119
130
|
|
|
120
131
|
attr_accessor :path_dir_state
|
|
121
132
|
|
|
122
|
-
def initialize(path_dir_schedules: nil, path_dir_state: nil)
|
|
133
|
+
def initialize(path_dir_schedules: nil, path_dir_state: nil, f_digest: nil)
|
|
123
134
|
@logger = Logger.new(STDOUT)
|
|
124
135
|
@path_dir_state = path_dir_state
|
|
125
136
|
@schedules = {}
|
|
@@ -128,6 +139,7 @@ module OAISchedules
|
|
|
128
139
|
unless path_dir_schedules.nil?
|
|
129
140
|
set_listener_dir_schedules(path_dir_schedules)
|
|
130
141
|
end
|
|
142
|
+
@f_digest = f_digest
|
|
131
143
|
end
|
|
132
144
|
|
|
133
145
|
|
|
@@ -180,6 +192,9 @@ module OAISchedules
|
|
|
180
192
|
state = {}
|
|
181
193
|
if File.file?(path_file_state)
|
|
182
194
|
state = read_state_file(path_file_state)
|
|
195
|
+
else
|
|
196
|
+
init_schedule_state(state)
|
|
197
|
+
write_state_file(path_file_state, state)
|
|
183
198
|
end
|
|
184
199
|
# create task
|
|
185
200
|
task = Concurrent::TimerTask.new(run_now: false) {
|
|
@@ -193,7 +208,7 @@ module OAISchedules
|
|
|
193
208
|
task.add_observer(TaskObserver.new(@logger, name))
|
|
194
209
|
# add item to schedules
|
|
195
210
|
@schedules[name] = {
|
|
196
|
-
content: content,
|
|
211
|
+
content: deep_copy(content),
|
|
197
212
|
task: task,
|
|
198
213
|
state_machine: StateMachineHarvesting.new,
|
|
199
214
|
state: state
|
|
@@ -212,7 +227,8 @@ module OAISchedules
|
|
|
212
227
|
|
|
213
228
|
|
|
214
229
|
def modify_schedule(name, content)
|
|
215
|
-
|
|
230
|
+
handle_schedule_state_at_schedule_change(name, content)
|
|
231
|
+
@schedules[name][:content] = deep_copy(content)
|
|
216
232
|
handle_schedule_task(name)
|
|
217
233
|
end
|
|
218
234
|
|
|
@@ -222,12 +238,44 @@ module OAISchedules
|
|
|
222
238
|
private
|
|
223
239
|
|
|
224
240
|
|
|
241
|
+
def init_schedule_state(state)
|
|
242
|
+
state["use_resumption_token"] = nil
|
|
243
|
+
state["resumption_token"] = nil
|
|
244
|
+
state["expiration_date_resumption_token"] = nil
|
|
245
|
+
state["datetime_now"] = get_datetime_now
|
|
246
|
+
state["datetime_next_harvesting"] = nil
|
|
247
|
+
state["count_success"] = 0
|
|
248
|
+
state["count_fails"] = 0
|
|
249
|
+
state["done"] = false
|
|
250
|
+
state["count_harvested_records"] = 0
|
|
251
|
+
state["latest_harvested_records_datestamp"] = nil
|
|
252
|
+
state["harvesting"] = false
|
|
253
|
+
state["error"] = nil
|
|
254
|
+
state["status"] = nil
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def handle_schedule_state_at_schedule_change(name, content)
|
|
259
|
+
state = @schedules[name][:state]
|
|
260
|
+
if (content["format"] != @schedules[name][:content]["format"]) \
|
|
261
|
+
|| (content["set"] != @schedules[name][:content]["set"]) \
|
|
262
|
+
|| (content["from"] != @schedules[name][:content]["from"]) \
|
|
263
|
+
|| (content["until"] != @schedules[name][:content]["until"])
|
|
264
|
+
init_schedule_state(state)
|
|
265
|
+
end
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def deep_copy(o)
|
|
270
|
+
Marshal.load(Marshal.dump(o))
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
|
|
225
274
|
def handle_schedule_task(name)
|
|
226
275
|
task = @schedules[name][:task]
|
|
227
276
|
interval_s_safe = 60
|
|
228
277
|
begin
|
|
229
|
-
|
|
230
|
-
interval_s = duration.to_seconds
|
|
278
|
+
interval_s = interval_iso8601_to_seconds(@schedules[name][:content]["interval"])
|
|
231
279
|
@logger.info("#{name}: task interval (s): #{interval_s}")
|
|
232
280
|
th_interval_s = 0.1 # protects from negative, 0 or small time intervals
|
|
233
281
|
if interval_s < th_interval_s
|
|
@@ -305,7 +353,9 @@ module OAISchedules
|
|
|
305
353
|
|
|
306
354
|
|
|
307
355
|
def logic(name, content, state_machine, state)
|
|
356
|
+
state["datetime_next_harvesting"] = get_datetime_next_schedule_tick_from_now(content["interval"])
|
|
308
357
|
loop do
|
|
358
|
+
# sleep(3)
|
|
309
359
|
@logger.info("#{name}: handling state: #{state_machine.state}")
|
|
310
360
|
case state_machine.state
|
|
311
361
|
when StateHarvesting::NOT_IDENTIFIED
|
|
@@ -317,51 +367,115 @@ module OAISchedules
|
|
|
317
367
|
)
|
|
318
368
|
state["identify"] = data
|
|
319
369
|
state_machine.add_event(EventHarvesting::DONE_IDENTIFY)
|
|
320
|
-
break
|
|
321
370
|
when StateHarvesting::IDLE
|
|
322
371
|
state_machine.add_event(EventHarvesting::REQUEST_HARVEST)
|
|
323
372
|
when StateHarvesting::HARVESTING
|
|
324
|
-
|
|
325
|
-
|
|
373
|
+
# use_resumption_token = false
|
|
374
|
+
use_resumption_token = true
|
|
375
|
+
state["use_resumption_token"] = use_resumption_token
|
|
376
|
+
format = content["format"] || ""
|
|
377
|
+
fmt_dt = content.dig("repository", "granularity")
|
|
378
|
+
from = convert_datetime(content["from"], fmt_dt) || ""
|
|
379
|
+
to = convert_datetime(content["until"], fmt_dt) || ""
|
|
380
|
+
set = content["set"] || ""
|
|
381
|
+
resumption_token = ""
|
|
382
|
+
if use_resumption_token
|
|
383
|
+
if !state["resumption_token"].nil?
|
|
384
|
+
# if state["expiration_date_resumption_token"].nil? || (!state["expiration_date_resumption_token"].nil? && (DateTime.parse(state["expiration_date_resumption_token"]) - DateTime.now) > 0)
|
|
385
|
+
format = ""
|
|
386
|
+
from = ""
|
|
387
|
+
to = ""
|
|
388
|
+
set = ""
|
|
389
|
+
resumption_token = state["resumption_token"]
|
|
390
|
+
# end
|
|
391
|
+
end
|
|
392
|
+
else
|
|
393
|
+
from = add_eps_to_datetime(state["latest_harvested_records_datestamp"], fmt_dt) || from
|
|
326
394
|
end
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
395
|
+
data = nil
|
|
396
|
+
error = nil
|
|
397
|
+
err_info = nil
|
|
398
|
+
harvesting = true
|
|
399
|
+
done = false
|
|
400
|
+
status = nil
|
|
401
|
+
begin
|
|
402
|
+
data = oai_get_records(
|
|
403
|
+
name,
|
|
404
|
+
content["repository"]["uri"],
|
|
405
|
+
format,
|
|
406
|
+
from,
|
|
407
|
+
to,
|
|
408
|
+
set,
|
|
409
|
+
resumption_token
|
|
410
|
+
)
|
|
411
|
+
state["resumption_token"] = nil
|
|
412
|
+
state["expiration_date_resumption_token"] = nil
|
|
413
|
+
if use_resumption_token
|
|
414
|
+
if data["resumptionToken"].is_a?(String)
|
|
415
|
+
state["resumption_token"] = data["resumptionToken"]
|
|
416
|
+
elsif data["resumptionToken"].is_a?(Hash)
|
|
417
|
+
data_token = data["resumptionToken"]
|
|
418
|
+
state["resumption_token"] = data_token["$text"]
|
|
419
|
+
state["expiration_date_resumption_token"] = data_token["_expirationDate"]
|
|
420
|
+
end
|
|
421
|
+
end
|
|
422
|
+
n_records = data["record"].size
|
|
423
|
+
state["count_harvested_records"] += n_records
|
|
424
|
+
if n_records > 0
|
|
425
|
+
timestamps = data["record"].map do |record|
|
|
426
|
+
record["header"]["datestamp"]
|
|
427
|
+
end.sort
|
|
428
|
+
timestamp_latest = timestamps[-1].strftime('%FT%TZ')
|
|
429
|
+
state["latest_harvested_records_datestamp"] = convert_datetime(timestamp_latest, fmt_dt)
|
|
430
|
+
end
|
|
431
|
+
state["count_success"] += 1
|
|
432
|
+
rescue StandardError => e
|
|
433
|
+
state["count_fails"] += 1
|
|
434
|
+
error = e
|
|
435
|
+
end
|
|
436
|
+
if error.nil?
|
|
437
|
+
err_info = nil
|
|
438
|
+
if use_resumption_token
|
|
439
|
+
if state["resumption_token"].nil?
|
|
440
|
+
state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
|
|
441
|
+
harvesting = false
|
|
442
|
+
status = "next harvesting: #{state["datetime_next_harvesting"]}"
|
|
443
|
+
else
|
|
444
|
+
state_machine.add_event(EventHarvesting::DONE_HARVEST)
|
|
445
|
+
harvesting = true
|
|
446
|
+
status = "harvesting"
|
|
447
|
+
end
|
|
448
|
+
else
|
|
449
|
+
state_machine.add_event(EventHarvesting::DONE_HARVEST)
|
|
450
|
+
harvesting = true
|
|
451
|
+
status = "harvesting"
|
|
452
|
+
end
|
|
333
453
|
else
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
454
|
+
err_info = {
|
|
455
|
+
"message" => error.message,
|
|
456
|
+
"backtrace" => error.backtrace
|
|
457
|
+
}
|
|
458
|
+
state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
|
|
459
|
+
harvesting = false
|
|
460
|
+
status = "error (see logs), next harvesting: #{state["datetime_next_harvesting"]}"
|
|
340
461
|
end
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
to,
|
|
347
|
-
set,
|
|
348
|
-
resumption_token
|
|
349
|
-
)
|
|
350
|
-
state["resumption_token"] = data["resumptionToken"]
|
|
351
|
-
state["count_partial_harversting"] += 1
|
|
462
|
+
state["done"] = done
|
|
463
|
+
state["harvesting"] = harvesting
|
|
464
|
+
state["datetime_now"] = get_datetime_now
|
|
465
|
+
state["status"] = status
|
|
466
|
+
state["error"] = err_info
|
|
352
467
|
path_file_state = get_path_state_file_from_schedule_name(name)
|
|
353
468
|
@logger.info("#{name}: writing to state file #{path_file_state}")
|
|
354
469
|
write_state_file(path_file_state, state)
|
|
355
|
-
|
|
356
|
-
state_machine.add_event(EventHarvesting::DONE_HARVEST)
|
|
357
|
-
else
|
|
358
|
-
state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
|
|
359
|
-
end
|
|
360
|
-
break
|
|
470
|
+
@f_digest&.call(name, content, data, done, error, state, @logger)
|
|
361
471
|
when StateHarvesting::COMPLETE
|
|
362
472
|
@logger.warn("#{name}: full harvesting complete")
|
|
363
|
-
|
|
364
|
-
|
|
473
|
+
state_machine.add_event(EventHarvesting::RESTART)
|
|
474
|
+
auto_deactivate_schedule = false
|
|
475
|
+
if auto_deactivate_schedule
|
|
476
|
+
content["active"] = false
|
|
477
|
+
handle_schedule_task(name)
|
|
478
|
+
end
|
|
365
479
|
break
|
|
366
480
|
else
|
|
367
481
|
@logger.warn("#{name}: state #{state_machine.state} not known")
|
|
@@ -375,11 +489,19 @@ module OAISchedules
|
|
|
375
489
|
verb = "Identify"
|
|
376
490
|
url_query = "#{url_base}?verb=#{verb}"
|
|
377
491
|
@logger.info("#{name}: fetching from #{url_query}")
|
|
378
|
-
|
|
492
|
+
input = DataCollector::Input.new.from_uri(url_query)
|
|
493
|
+
if input.nil?
|
|
494
|
+
raise StandardError, "#{url_query}, service not found"
|
|
495
|
+
end
|
|
496
|
+
data = DataCollector::Core.filter(input, "$..#{verb}")
|
|
379
497
|
if data.empty?
|
|
380
|
-
|
|
498
|
+
error = DataCollector::Core.filter(input, "$..error")
|
|
499
|
+
error = error[0]
|
|
500
|
+
name_error = error['_code']
|
|
501
|
+
text_error = error['$text']
|
|
502
|
+
raise StandardError, "#{name}: #{url_query}: #{name_error}, #{text_error}"
|
|
381
503
|
end
|
|
382
|
-
data
|
|
504
|
+
data[0]
|
|
383
505
|
rescue DataCollector::InputError => e
|
|
384
506
|
raise RuntimeError, "#{name}: #{url_query} not found, or server error"
|
|
385
507
|
rescue StandardError => e
|
|
@@ -406,11 +528,22 @@ module OAISchedules
|
|
|
406
528
|
url_query += "&resumptionToken=#{resumption_token}"
|
|
407
529
|
end
|
|
408
530
|
@logger.info("#{name}: fetching from #{url_query}")
|
|
409
|
-
|
|
531
|
+
input = DataCollector::Input.new.from_uri(url_query)
|
|
532
|
+
if input.nil?
|
|
533
|
+
raise StandardError, "#{url_query}, service not found"
|
|
534
|
+
end
|
|
535
|
+
data = DataCollector::Core.filter(input, "$..#{verb}")
|
|
410
536
|
if data.empty?
|
|
411
|
-
|
|
537
|
+
error = DataCollector::Core.filter(input, "$..error")
|
|
538
|
+
error = error[0]
|
|
539
|
+
name_error = error['_code']
|
|
540
|
+
text_error = error['$text']
|
|
541
|
+
raise StandardError, "#{name}: #{url_query}: #{name_error}, #{text_error}"
|
|
412
542
|
end
|
|
413
|
-
|
|
543
|
+
if data.compact.empty?
|
|
544
|
+
raise StandardError, "#{name}: #{url_query}: no records found"
|
|
545
|
+
end
|
|
546
|
+
data[0]
|
|
414
547
|
rescue DataCollector::InputError => e
|
|
415
548
|
raise RuntimeError, "#{name}: #{url_query} not found, or server error"
|
|
416
549
|
rescue StandardError => e
|
|
@@ -443,6 +576,48 @@ module OAISchedules
|
|
|
443
576
|
modify_schedule(name_schedule, content_schedule)
|
|
444
577
|
end
|
|
445
578
|
|
|
579
|
+
def convert_datetime(str_dt, fmt_dt)
|
|
580
|
+
return nil unless str_dt
|
|
581
|
+
return str_dt unless fmt_dt
|
|
582
|
+
dt = DateTime.parse(str_dt)
|
|
583
|
+
case fmt_dt
|
|
584
|
+
when 'YYYY-MM-DD'
|
|
585
|
+
dt.strftime('%F')
|
|
586
|
+
when 'YYYY-MM-DDThh:mm:ssZ'
|
|
587
|
+
dt.strftime('%FT%TZ')
|
|
588
|
+
else
|
|
589
|
+
str_dt
|
|
590
|
+
end
|
|
591
|
+
end
|
|
592
|
+
|
|
593
|
+
def add_eps_to_datetime(str_dt, fmt_dt)
|
|
594
|
+
return nil unless str_dt
|
|
595
|
+
dt = DateTime.parse(str_dt)
|
|
596
|
+
case fmt_dt
|
|
597
|
+
when 'YYYY-MM-DD'
|
|
598
|
+
dt += 1.0
|
|
599
|
+
dt.strftime('%F')
|
|
600
|
+
when 'YYYY-MM-DDThh:mm:ssZ'
|
|
601
|
+
dt += 1.0/(24*60*60)
|
|
602
|
+
dt.strftime('%FT%TZ')
|
|
603
|
+
end
|
|
604
|
+
end
|
|
605
|
+
|
|
606
|
+
def get_datetime_now
|
|
607
|
+
Time.current.in_time_zone('Zulu').strftime('%FT%TZ')
|
|
608
|
+
end
|
|
609
|
+
|
|
610
|
+
def interval_iso8601_to_seconds(str_interval)
|
|
611
|
+
ISO8601::Duration.new(str_interval).to_seconds
|
|
612
|
+
end
|
|
613
|
+
|
|
614
|
+
def get_datetime_next_schedule_tick_from_now(str_interval)
|
|
615
|
+
interval_s = interval_iso8601_to_seconds(str_interval)
|
|
616
|
+
dt = Time.current.in_time_zone('Zulu')
|
|
617
|
+
dt += interval_s
|
|
618
|
+
dt.strftime('%FT%TZ')
|
|
619
|
+
end
|
|
620
|
+
|
|
446
621
|
end
|
|
447
622
|
|
|
448
623
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,28 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: oai_schedules
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1
|
|
4
|
+
version: 0.9.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Davide Monari
|
|
8
8
|
bindir: exe
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date: 2025-
|
|
10
|
+
date: 2025-07-15 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: activesupport
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - "~>"
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: 8.0.2
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - "~>"
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: 8.0.2
|
|
12
26
|
- !ruby/object:Gem::Dependency
|
|
13
27
|
name: concurrent-ruby
|
|
14
28
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -65,7 +79,21 @@ dependencies:
|
|
|
65
79
|
- - "~>"
|
|
66
80
|
- !ruby/object:Gem::Version
|
|
67
81
|
version: 3.9.0
|
|
68
|
-
|
|
82
|
+
- !ruby/object:Gem::Dependency
|
|
83
|
+
name: tzinfo-data
|
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - "~>"
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: 1.2025.2
|
|
89
|
+
type: :runtime
|
|
90
|
+
prerelease: false
|
|
91
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
92
|
+
requirements:
|
|
93
|
+
- - "~>"
|
|
94
|
+
- !ruby/object:Gem::Version
|
|
95
|
+
version: 1.2025.2
|
|
96
|
+
description: gem to run concurrent OAI-PMH harvesting schedules
|
|
69
97
|
email:
|
|
70
98
|
- davide.monari@kuleuven.be
|
|
71
99
|
executables: []
|
|
@@ -78,7 +106,15 @@ files:
|
|
|
78
106
|
- LICENSE.txt
|
|
79
107
|
- README.md
|
|
80
108
|
- Rakefile
|
|
81
|
-
- examples/dir_schedules
|
|
109
|
+
- examples/dir_schedules/.gitkeep
|
|
110
|
+
- examples/dir_schedules_all/schedule_495607cf-f773-463f-8ee1-77d0f53e0c29.json
|
|
111
|
+
- examples/dir_schedules_all/schedule_5e344861-806b-4361-98f7-a0be6a5984de.json
|
|
112
|
+
- examples/dir_schedules_all/schedule_712f46ab-f87f-4db8-b69e-7101d9e2ae61.json
|
|
113
|
+
- examples/dir_schedules_all/schedule_99922363-5b37-4438-a274-5a4a5167f811.json
|
|
114
|
+
- examples/dir_schedules_all/schedule_dc34623d-2ae2-4e90-91d8-26f4ba29a056.json
|
|
115
|
+
- examples/dir_schedules_all/schedule_deventit.json
|
|
116
|
+
- examples/dir_schedules_all/schedule_heron.json
|
|
117
|
+
- examples/dir_schedules_all/schedule_sample.json
|
|
82
118
|
- examples/dir_state/.gitkeep
|
|
83
119
|
- examples/example_01.rb
|
|
84
120
|
- examples/example_02.rb
|
|
@@ -110,5 +146,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
110
146
|
requirements: []
|
|
111
147
|
rubygems_version: 3.6.2
|
|
112
148
|
specification_version: 4
|
|
113
|
-
summary: gem to run concurrent OAI-
|
|
149
|
+
summary: gem to run concurrent OAI-PMH harvesting schedules
|
|
114
150
|
test_files: []
|