oai_schedules 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f254ace930dec7e151a5af485130dd23623548d6f9d0860a764b90ce217c4bb
-  data.tar.gz: bfe93c7f19a894976c53f3d2e3d720e048c77c116b92283034ba7b17d4efe386
+  metadata.gz: d2b2bae058157785c775ebd1b8c0ea76e8685adb8588280f6c6427644b9da8b5
+  data.tar.gz: 928102f14abd5bf59bb7b55bb7e100074c5b20bec3276e1dd129a04e74c6da60
 SHA512:
-  metadata.gz: bbd9df7881c619f2660dd6b622adf2fd7ec6fa35cde400ff3490b1001b3635c7d89605882e53c9b242cc992de6d0153a7ec5d25d5b41e0672853b997924900a0
-  data.tar.gz: 918414787e68f89b8dab0e692096e708864ca7b32a81d1341c0cc09e72fbb70d2cd886a137f024e5946b6176b7d4b1b20d606ec12d33c81894c65588ca9c712e
+  metadata.gz: 35bb16497f382eb1accaecd95e1e2b022f3addeeef7a4ae79ab5bc4b9f6b023f3fbee70535815ee0edcf5e48b62fc4bdb697cba378cdbac16f114420394cf85e
+  data.tar.gz: 3811fd28be558414ebfabcc83e1978080578ff4b33147f0da06cf569ab6b42af498489cf7bef3dd5f14b6e0dbb8598dc615d15220c7496c710f93fcea0149a3e

data/lib/oai_schedules/manager.rb

@@ -181,6 +181,9 @@ module OAISchedules
       state = {}
       if File.file?(path_file_state)
         state = read_state_file(path_file_state)
+      else
+        init_schedule_state(state)
+        write_state_file(path_file_state, state)
       end
       # create task
       task = Concurrent::TimerTask.new(run_now: false) {
@@ -224,14 +227,22 @@ module OAISchedules
     private
 
 
+    def init_schedule_state(state)
+      state["resumption_token"] = nil
+      state["count_success"] = 0
+      state["count_fails"] = 0
+      state["done"] = false
+      state["count_harvested_records"] = 0
+      state["latest_harvested_records_datestamp"] = ""
+    end
+
+
     def handle_schedule_state_at_schedule_change(name, content)
       # invalidate resumption token if either schedule format or set changes
       state = @schedules[name][:state]
       if    (content["format"] != @schedules[name][:content]["format"]) \
         ||  (content["set"] != @schedules[name][:content]["set"])
-        state["resumption_token"] = nil
-        state["count_success"] = 0
-        state["count_fails"] = 0
+        init_schedule_state(state)
       end
     end
 
@@ -346,6 +357,15 @@ module OAISchedules
           unless state.has_key?("count_fails")
             state["count_fails"] = 0
           end
+          unless state.has_key?("done")
+            state["done"] = false
+          end
+          unless state.has_key?("count_harvested_records")
+            state["count_harvested_records"] = 0
+          end
+          unless state.has_key?("latest_harvested_records_datestamp")
+            state["latest_harvested_records_datestamp"] = ""
+          end
           if !state["resumption_token"].nil?
             format = ""
             from = ""
@@ -361,35 +381,49 @@ module OAISchedules
           end
           data = nil
           error = nil
-          begin
-            data = oai_get_records(
-              name,
-              content["repository"]["uri"],
-              format,
-              from,
-              to,
-              set,
-              resumption_token
-            )
-            state["resumption_token"] = data["resumptionToken"]
-            state["count_success"] += 1
-          rescue StandardError => e
-            state["count_fails"] += 1
-            error = e
+          done = false
+          if state["done"]
+            state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
+            done = true
+          else
+            begin
+              data = oai_get_records(
+                name,
+                content["repository"]["uri"],
+                format,
+                from,
+                to,
+                set,
+                resumption_token
+              )
+              state["resumption_token"] = data["resumptionToken"]
+              state["count_success"] += 1
+              n_records = data["record"].size
+              state["count_harvested_records"] += n_records
+              if n_records > 0
+                timestamps = data["record"].map do |record|
+                  record["header"]["datestamp"]
+                end.sort
+                state["latest_harvested_records_datestamp"] = timestamps[-1]
+              end
+            rescue StandardError => e
+              state["count_fails"] += 1
+              error = e
+            end
+            if error.nil?
+              if !data["resumptionToken"].nil?
+                state_machine.add_event(EventHarvesting::DONE_HARVEST)
+                done = false
+              else
+                state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
+                done = true
+              end
+            end
+            state["done"] = done
           end
           path_file_state = get_path_state_file_from_schedule_name(name)
           @logger.info("#{name}: writing to state file #{path_file_state}")
           write_state_file(path_file_state, state)
-          done = false
-          if error.nil?
-            if !data["resumptionToken"].nil?
-              state_machine.add_event(EventHarvesting::DONE_HARVEST)
-              done = false
-            else
-              state_machine.add_event(EventHarvesting::DONE_FULL_HARVEST)
-              done = true
-            end
-          end
           @f_digest&.call(name, content, data, done, error, state, @logger)
           break
         when StateHarvesting::COMPLETE

data/lib/oai_schedules/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OaiSchedules
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oai_schedules
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Davide Monari