fair_champion_harvester 0.1.16 → 0.1.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ebd706f0e7aea904a19364640429bdec400a208ce4503f363f7d6bc8924351b7
-  data.tar.gz: f7d1def76e7b3b5715a14c4f90d107f2fe23154aa9a63c7c9c7892cd71fce26c
+  metadata.gz: f36986c04c73a293f76a80e6415296024153b1c2c756bd8777f67d17c05e0a32
+  data.tar.gz: c2e22acdc3c74da575c380e9eb9f7be40b0594f6a78525ead0ff39d3fdf3149f
 SHA512:
-  metadata.gz: 03121c8e7bfd86f47a50ef007a1382d0a063d8ae5bed9aefe1144b131d64524c50a8d9ad372ccf74dd26fa594f6d474473d4d736cb072406e415d5db086f3f8b
-  data.tar.gz: e61abd8e1212cd453196c2f83d096f60d70b330362aa20ca5c934b86007896ad9ce17ecae68e1210b0ac830ee276fdd089ad599c769c44b75616d9fa014d6e4d
+  metadata.gz: 7e8075969817d032ca9e84dfe02260981eb78043390ddfdb415019ea102e1aa549b9828d68b13661e34f3c83f2da32f1b89a2e2091f4ab9dec6d98e47d68e389
+  data.tar.gz: 223e4cf7c16da80ba9b2756db342c2398e97cf7f05b31169add185c4a4f5e331663502a6fd0e1e89114b9ba8d42c9f8b9776b32b58aaff0b69a273790bf6396a

data/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [0.1.17] - 2026-08-11
+
+### Fixed
+
+- `Cache.checkCache` and `Cache.checkRDFCache` (`lib/cache.rb`) never expired anything — once a `/tmp/<md5>_*` entry was written, `harvester.rb#fetch` returned it forever, since `/tmp` inside the `communitytests` container is not a persistent volume and the only ways to clear a stale entry were a full container restart or manually deleting files over SSH. Added a `cached_at` timestamp file per entry (and per RDF graph cache entry) plus a TTL check on read, configurable via `CACHE_TTL` (default 300s / 5 minutes — short by design, since harvested DCAT/TTL records under test change frequently and testers need to see edits reflected quickly, unlike FDP-Index-Proxy's 24h `FDP_CACHE_TTL`). Also added `Cache.purge(uri, headers)` to force-evict a specific entry without waiting for TTL expiry.
+
 ## [0.1.16] - 2026-08-05
 
 ### Fixed

data/lib/cache.rb

@@ -4,14 +4,34 @@ module FAIRChampionHarvester
     ###################  CACHE FUNCTIONS #####################
     ###################  #####################################
 
+    # Cache time-to-live in seconds. Configurable via the +CACHE_TTL+ env
+    # var; defaults to 5 minutes (short by design, unlike FDP-Index-Proxy's
+    # 24h FDP_CACHE_TTL, since harvested DCAT/TTL records under test change
+    # frequently and testers need to see edits reflected quickly).
+    CACHE_TTL = (ENV.fetch("CACHE_TTL", 300)).to_i
+
+    def self.expired?(timestamp_file)
+      return true unless File.exist?(timestamp_file)
+
+      age = Time.now - Marshal.load(File.read(timestamp_file))
+      age > CACHE_TTL
+    end
+
     def self.checkRDFCache(body)
       g = RDF::Graph.new
       key = Digest::MD5.hexdigest body
       graph_file = "/tmp/#{key}_graph"
       body_file  = "/tmp/#{key}_graphbody"
+      time_file  = "/tmp/#{key}_graphtime"
 
       return g unless File.exist?(graph_file) && File.exist?(body_file)
 
+      if expired?(time_file)
+        warn "RDF Cache File #{key} EXPIRED"
+        purgeRDFFiles(key)
+        return g
+      end
+
       warn "RDF Cache File #{key} FOUND"
       graph = Marshal.load(File.read(graph_file))
       graph.each { |statement| g << statement }
@@ -26,17 +46,33 @@ module FAIRChampionHarvester
       warn "WRITING RDF TO CACHE #{filename}"
       File.binwrite("/tmp/#{filename}_graph", Marshal.dump(graph))
       File.binwrite("/tmp/#{filename}_graphbody", body)
+      File.binwrite("/tmp/#{filename}_graphtime", Marshal.dump(Time.now))
       warn "wrote RDF filename: #{filename}"
     end
 
     def self.checkCache(uri, headers)
       filename = Digest::MD5.hexdigest uri + headers.to_s
+      time_file = "/tmp/#{filename}_time"
       warn "Checking Error cache for #{filename}"
+
       if File.exist?("/tmp/#{filename}_error")
-        warn "Error file found in cache... returning"
-        return ["ERROR", nil, [uri]]
+        if expired?(time_file)
+          warn "Error cache file #{filename} EXPIRED"
+          purgeFiles(filename)
+        else
+          warn "Error file found in cache... returning"
+          return ["ERROR", nil, [uri]]
+        end
       end
+
       if File.exist?("/tmp/#{filename}_head") and File.exist?("/tmp/#{filename}_body")
+        if expired?(time_file)
+          warn "Cache #{filename} EXPIRED"
+          purgeFiles(filename)
+          warn "Not Found in Cache"
+          return
+        end
+
         warn "FOUND data in cache"
         head = Marshal.load(File.read("/tmp/#{filename}_head"))
         body = Marshal.load(File.read("/tmp/#{filename}_body"))
@@ -54,15 +90,42 @@ module FAIRChampionHarvester
       headfilename = filename + "_head"
       bodyfilename = filename + "_body"
       urifilename = filename + "_uri"
+      timefilename = filename + "_time"
       File.binwrite("/tmp/#{headfilename}", Marshal.dump(head))
       File.binwrite("/tmp/#{bodyfilename}", Marshal.dump(body))
       File.binwrite("/tmp/#{urifilename}", Marshal.dump(finalURI))
+      File.binwrite("/tmp/#{timefilename}", Marshal.dump(Time.now))
     end
 
     def self.writeErrorToCache(uri, headers)
       filename = Digest::MD5.hexdigest uri + headers.to_s
       warn "in writeErrorToCache Writing error to cache for #{filename}"
       File.binwrite("/tmp/#{filename}_error", "ERROR")
+      File.binwrite("/tmp/#{filename}_time", Marshal.dump(Time.now))
+    end
+
+    # Force-evicts the cache entry (data and/or error) for a given uri+headers,
+    # regardless of TTL. Use this to bypass a stale or bad cached result
+    # without waiting for CACHE_TTL to elapse or restarting the container.
+    def self.purge(uri, headers)
+      filename = Digest::MD5.hexdigest uri + headers.to_s
+      purgeFiles(filename)
+    end
+
+    def self.purgeFiles(filename)
+      %w[_head _body _uri _error _time].each do |suffix|
+        path = "/tmp/#{filename}#{suffix}"
+        File.delete(path) if File.exist?(path)
+      end
+    end
+    private_class_method :purgeFiles
+
+    def self.purgeRDFFiles(key)
+      %w[_graph _graphbody _graphtime].each do |suffix|
+        path = "/tmp/#{key}#{suffix}"
+        File.delete(path) if File.exist?(path)
+      end
     end
+    private_class_method :purgeRDFFiles
   end
 end

data/lib/fair_champion_harvester/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FairChampionHarvester
-  VERSION = "0.1.16"
+  VERSION = "0.1.17"
 end

data/lib/harvester.rb

@@ -1,4 +1,4 @@
-HARVESTER_VERSION = "Hvst-0.1.16".freeze
+HARVESTER_VERSION = "Hvst-0.1.17".freeze
 # better output,
 # different dealing with DataCite (they have a unique type header)
 # handle large extruct output,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fair_champion_harvester
 version: !ruby/object:Gem::Version
-  version: 0.1.16
+  version: 0.1.17
 platform: ruby
 authors:
 - markwilkinson