relaton-w3c 2.1.2 → 2.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46f27195b30d4b285034731e6749b4ed4aff9ed607d41530a1d5b9d07c859b22
-  data.tar.gz: a263250dc4bda6a871af9addecbe37c5e3d62ab748797fe064dc9545e90eec8a
+  metadata.gz: ac21e91a675f1a0c33ea6a0610edb38895ffe87d9b810593331f9a79cdbe0324
+  data.tar.gz: 74993548a097428280e01147eea4dd604adf181ed33c9d55b1db33e8a994c489
 SHA512:
-  metadata.gz: 973bcf91864d27cb1f19f6001733dc0b46c66738d1e6614ff91313a61dcf03b6a721cbc8c82fb3bd64a00dccc8af50dd2efbf23d051ada503d4b8c698719bb68
-  data.tar.gz: 89fb5bcce023f69488932d901fbe260450bf1fcd6164c46be7731fb107bcb47a9aec30b9eb08a7fe534822727dcdb48c262e2386cf7f921ce018f0537fb33377
+  metadata.gz: 6851b1f389210dfe5bbef588b1d9de910c7f0d8c595e896a1d3419a937d20bbc643dbe08af40bd896e9f8f8149f59bea736b2f338512046f6d11f197e21b0999
+  data.tar.gz: 6a6db0027bde29086eff8f66240a9343288b393e2c25515e6e92b667d31b0fe89ea100de94adbb4bd8d244f7b527e7d05b3be1edf97917fad54512a88ca79132

data/CLAUDE.md

@@ -49,7 +49,7 @@ All classes live under `lib/relaton/w3c/` in the `Relaton::W3c` namespace:
 **Data fetching:**
 - **`DataFetcher`** (`data_fetcher.rb`) — extends `Core::DataFetcher`, fetches all W3C specs via the W3C API
 - **`DataParser`** (`data_parser.rb`) — converts W3C API spec objects into `Relaton::W3c::Item` instances
-- **`RateLimitHandler`** (`rate_limit_handler.rb`) — mixin for retry logic and caching of fetched API objects
+- **`SafeRealize`** (`safe_realize.rb`) — mixin that, on a terminal error, skips the resource (returns `nil`) so one bad link doesn't abort the crawl (see Rate limiting & retries). It does not retry or cache successes — those live upstream.
 - **`PubId`** (`pubid.rb`) — parses and compares W3C document identifiers (stage, code, date parts)
 
 **Utilities:**
@@ -57,13 +57,22 @@ All classes live under `lib/relaton/w3c/` in the `Relaton::W3c` namespace:
 
 The entry module is defined in `lib/relaton/w3c.rb` and exposes `grammar_hash`.
 
+### Rate limiting & retries
+
+Transient-failure resilience is layered upstream, not in this gem:
+- **w3c_api** builds its HAL client with `faraday-retry` to retry HTTP 403 (the W3C rate-limit signal) and connection/timeout errors.
+- **lutaml-hal** (beneath w3c_api) retries 429 and 5xx with exponential backoff.
+
+Successful objects are cached by **w3c_api** (lutaml-hal caches realized objects keyed by URL, thread-safely as of lutaml-hal 0.2.1), so `SafeRealize` doesn't cache them. It only **retries nothing** and remembers hrefs that failed terminally (in a `Concurrent::Map`), returning `nil` for them so one bad link doesn't abort the crawl and isn't re-fetched on every reference. Network errors are not remembered, so a later reference can try again.
+
 ### Key Dependencies
 
-- **relaton-bib** (~> 2.0.0-alpha) — provides base `Bib::Item`, `Bib::Ext`, `Bib::Doctype` and serialization mixins (LutaML model layer)
+- **relaton-bib** (~> 2.1.0) — provides base `Bib::Item`, `Bib::Ext`, `Bib::Doctype` and serialization mixins (LutaML model layer)
 - **relaton-core** — provides base `Core::Processor` and `Core::DataFetcher`
-- **relaton-index** — index-based search for bibliographic references
-- **w3c_api** — W3C API client used by `DataFetcher` to retrieve specifications
-- **linkeddata/rdf/sparql** — legacy RDF dependencies (still in gemspec)
+- **relaton-index** — index-based search for bibliographic references; also unpacks the index zip at runtime
+- **w3c_api** (~> 0.3.2) — W3C API (HAL/REST) client used by `DataFetcher` to retrieve specifications; owns rate-limit and transient-error retries, and the (thread-safe) object cache
+
+The W3C data is fetched entirely through `w3c_api`; the older RDF/SPARQL/scraping stack (linkeddata, rdf, sparql, shex, mechanize, …) has been removed.
 
 ### Schema Validation
 
@@ -82,7 +91,7 @@ Tests use RSpec with:
 - **VCR** — recorded HTTP cassettes in `spec/vcr_cassettes/` (7-day re-record interval)
 - **WebMock** — disables external HTTP in tests
 
-Test fixtures live in `spec/fixtures/` (YAML, XML, RDF files).
+Test fixtures live in `spec/fixtures/` (YAML and XML files).
 
 ## Style
 

data/Gemfile

@@ -9,6 +9,7 @@ gem "rspec", "~> 3.0"
 
 gem "equivalent-xml"
 gem "ruby-jing"
+gem "rubyzip" # test-only: spec/support reads a zipped index fixture
 gem "simplecov"
 gem "vcr"
 gem "webmock"

data/lib/relaton/w3c/data_fetcher.rb

@@ -1,14 +1,14 @@
 require "relaton/core"
 require "w3c_api"
 require_relative "../w3c"
-require_relative "rate_limit_handler"
+require_relative "safe_realize"
 require_relative "data_parser"
 require_relative "pubid"
 
 module Relaton
   module W3c
     class DataFetcher < Core::DataFetcher
-      include Relaton::W3c::RateLimitHandler
+      include Relaton::W3c::SafeRealize
 
       DEFAULT_CONCURRENCY = 8
 
@@ -52,7 +52,12 @@ module Relaton
           specs.links.specifications.each { |spec| queue << spec }
           break unless specs.next?
 
-          specs = specs.next
+          # Route pagination through realize so transient 403/5xx on the
+          # next-page link retry with backoff instead of crashing the crawl.
+          next_page = realize(specs.links.next)
+          break unless next_page
+
+          specs = next_page
         end
 
         n_workers.times { queue << nil } # poison pills

data/lib/relaton/w3c/data_parser.rb

@@ -1,7 +1,7 @@
 module Relaton
   module W3c
     class DataParser
-      include Relaton::W3c::RateLimitHandler
+      include Relaton::W3c::SafeRealize
 
       USED_TYPES = %w[WD NOTE PER PR REC CR].freeze
 

data/lib/relaton/w3c/safe_realize.rb

@@ -0,0 +1,55 @@
+require "concurrent/map"
+
+module Relaton
+  module W3c
+    # Thin wrapper over lutaml-hal's `realize`. Successful objects are cached by
+    # w3c_api (it caches realized objects keyed by URL), so this only remembers
+    # resources that failed terminally and returns nil for them — so one broken
+    # link doesn't abort the crawl and isn't re-fetched on every reference.
+    #
+    # Transient failures are retried upstream: w3c_api retries HTTP 403 (the
+    # W3C rate-limit signal) and connection/timeout errors, and lutaml-hal
+    # retries 429 and 5xx. By the time an error surfaces here it is terminal.
+    module SafeRealize
+      # Hrefs that failed terminally — one map shared by every includer
+      # (DataFetcher and DataParser) since a broken resource is broken for the
+      # whole crawl. Initialized eagerly (at load, single-threaded) so the
+      # parallel fetcher's first concurrent access can't race a lazy `||=`;
+      # Concurrent::Map then handles the concurrent reads/writes.
+      @skipped = Concurrent::Map.new
+
+      def self.skipped
+        @skipped
+      end
+
+      def realize(obj)
+        href = resolve_href(obj)
+        return nil if SafeRealize.skipped.key?(href)
+
+        obj.realize
+      rescue Lutaml::Hal::ConnectionError, Lutaml::Hal::TimeoutError, Faraday::Error, Net::OpenTimeout => e
+        # Network-level failure (already retried by w3c_api). The resource itself
+        # is fine, so don't skip it permanently — a later reference can try again.
+        Util.warn "Failed to realize object: #{href}, error: #{e.message}"
+        nil
+      rescue Lutaml::Hal::NotFoundError
+        Util.warn "Object not found: #{href}"
+        SafeRealize.skipped[href] = true
+        nil
+      rescue Lutaml::Hal::Error => e
+        # Definitive upstream error (403 rate-limit, 5xx, 429) already retried by
+        # w3c_api / lutaml-hal. Skip the broken/unavailable resource rather than
+        # re-hitting it for every link that references it.
+        Util.warn "Skipping #{href}, upstream error after retries: #{e.message}"
+        SafeRealize.skipped[href] = true
+        nil
+      end
+
+      private
+
+      def resolve_href(obj)
+        obj.href || obj.links.self.href
+      end
+    end
+  end
+end

data/lib/relaton/w3c/version.rb

@@ -1,5 +1,5 @@
 module Relaton
   module W3c
-    VERSION = "2.1.2".freeze
+    VERSION = "2.1.3".freeze
   end
 end

data/relaton-w3c.gemspec

@@ -31,16 +31,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "linkeddata", "~> 3.2"
-  spec.add_dependency "mechanize", "~> 2.10"
-  spec.add_dependency "rdf", "~> 3.2"
-  spec.add_dependency "rdf-normalize", "~> 0.6"
   spec.add_dependency "relaton-bib", "~> 2.1.0"
   spec.add_dependency "relaton-core", "~> 0.0.13"
   spec.add_dependency "relaton-index", "~> 0.2.8"
-  spec.add_dependency "rubyzip", "~> 2.3"
-  spec.add_dependency "shex", "~> 0.7"
-  spec.add_dependency "csv", "~> 3.0"
-  spec.add_dependency "sparql", "~> 3.2"
-  spec.add_dependency "w3c_api", "~> 0.1.3"
+  spec.add_dependency "w3c_api", "~> 0.3.2"
 end

metadata

@@ -1,71 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: relaton-w3c
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.1.3
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-15 00:00:00.000000000 Z
+date: 2026-06-03 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: linkeddata
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-- !ruby/object:Gem::Dependency
-  name: mechanize
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.10'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.10'
-- !ruby/object:Gem::Dependency
-  name: rdf
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-- !ruby/object:Gem::Dependency
-  name: rdf-normalize
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
@@ -108,76 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.8
-- !ruby/object:Gem::Dependency
-  name: rubyzip
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.3'
-- !ruby/object:Gem::Dependency
-  name: shex
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.7'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.7'
-- !ruby/object:Gem::Dependency
-  name: csv
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: sparql
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: w3c_api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.3.2
 description: 'Relaton::W3c: retrieve W3C Standards for bibliographic using the IsoBibliographicItem
   model'
 email:
@@ -200,11 +88,6 @@ files:
 - bin/console
 - bin/rspec
 - bin/setup
-- grammars/basicdoc.rng
-- grammars/biblio-standoc.rng
-- grammars/biblio.rng
-- grammars/relaton-w3c-compile.rng
-- grammars/relaton-w3c.rng
 - lib/relaton/w3c.rb
 - lib/relaton/w3c/bibdata.rb
 - lib/relaton/w3c/bibitem.rb
@@ -217,7 +100,7 @@ files:
 - lib/relaton/w3c/item_data.rb
 - lib/relaton/w3c/processor.rb
 - lib/relaton/w3c/pubid.rb
-- lib/relaton/w3c/rate_limit_handler.rb
+- lib/relaton/w3c/safe_realize.rb
 - lib/relaton/w3c/util.rb
 - lib/relaton/w3c/version.rb
 - relaton-w3c.gemspec