relaton-w3c 2.1.1 → 2.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36c34b0daf914dbc2cf41cd4fe4842fc04d67147d977ac1282b9d7f0452538d7
-  data.tar.gz: f6f0fdf9774a4bdb798b4d00e0ba4dae18e6d9ec772ce80d6604d5666fb22893
+  metadata.gz: ac21e91a675f1a0c33ea6a0610edb38895ffe87d9b810593331f9a79cdbe0324
+  data.tar.gz: 74993548a097428280e01147eea4dd604adf181ed33c9d55b1db33e8a994c489
 SHA512:
-  metadata.gz: a6b3105b9b30556d46badf314f8c298718422c2cda7b9e015af0469cf64d99205238c7e786b1d7d429277ef2f58498241f99a2f170463ff273f94a53e1d58857
-  data.tar.gz: f51e0ee34082f78e51d0da502087ecd4ca7b7cb4e1903f7b2db20123ebddaf56d57c655c057825022df459f02e50ade7b1cbb7d3ad34aff4c621240aeebe6006
+  metadata.gz: 6851b1f389210dfe5bbef588b1d9de910c7f0d8c595e896a1d3419a937d20bbc643dbe08af40bd896e9f8f8149f59bea736b2f338512046f6d11f197e21b0999
+  data.tar.gz: 6a6db0027bde29086eff8f66240a9343288b393e2c25515e6e92b667d31b0fe89ea100de94adbb4bd8d244f7b527e7d05b3be1edf97917fad54512a88ca79132

data/CLAUDE.md

@@ -49,7 +49,7 @@ All classes live under `lib/relaton/w3c/` in the `Relaton::W3c` namespace:
 **Data fetching:**
 - **`DataFetcher`** (`data_fetcher.rb`) — extends `Core::DataFetcher`, fetches all W3C specs via the W3C API
 - **`DataParser`** (`data_parser.rb`) — converts W3C API spec objects into `Relaton::W3c::Item` instances
-- **`RateLimitHandler`** (`rate_limit_handler.rb`) — mixin for retry logic and caching of fetched API objects
+- **`SafeRealize`** (`safe_realize.rb`) — mixin that, on a terminal error, skips the resource (returns `nil`) so one bad link doesn't abort the crawl (see Rate limiting & retries). It does not retry or cache successes — those live upstream.
 - **`PubId`** (`pubid.rb`) — parses and compares W3C document identifiers (stage, code, date parts)
 
 **Utilities:**
@@ -57,13 +57,22 @@ All classes live under `lib/relaton/w3c/` in the `Relaton::W3c` namespace:
 
 The entry module is defined in `lib/relaton/w3c.rb` and exposes `grammar_hash`.
 
+### Rate limiting & retries
+
+Transient-failure resilience is layered upstream, not in this gem:
+- **w3c_api** builds its HAL client with `faraday-retry` to retry HTTP 403 (the W3C rate-limit signal) and connection/timeout errors.
+- **lutaml-hal** (beneath w3c_api) retries 429 and 5xx with exponential backoff.
+
+Successful objects are cached by **w3c_api** (lutaml-hal caches realized objects keyed by URL, thread-safely as of lutaml-hal 0.2.1), so `SafeRealize` doesn't cache them. It only **retries nothing** and remembers hrefs that failed terminally (in a `Concurrent::Map`), returning `nil` for them so one bad link doesn't abort the crawl and isn't re-fetched on every reference. Network errors are not remembered, so a later reference can try again.
+
 ### Key Dependencies
 
-- **relaton-bib** (~> 2.0.0-alpha) — provides base `Bib::Item`, `Bib::Ext`, `Bib::Doctype` and serialization mixins (LutaML model layer)
+- **relaton-bib** (~> 2.1.0) — provides base `Bib::Item`, `Bib::Ext`, `Bib::Doctype` and serialization mixins (LutaML model layer)
 - **relaton-core** — provides base `Core::Processor` and `Core::DataFetcher`
-- **relaton-index** — index-based search for bibliographic references
-- **w3c_api** — W3C API client used by `DataFetcher` to retrieve specifications
-- **linkeddata/rdf/sparql** — legacy RDF dependencies (still in gemspec)
+- **relaton-index** — index-based search for bibliographic references; also unpacks the index zip at runtime
+- **w3c_api** (~> 0.3.2) — W3C API (HAL/REST) client used by `DataFetcher` to retrieve specifications; owns rate-limit and transient-error retries, and the (thread-safe) object cache
+
+The W3C data is fetched entirely through `w3c_api`; the older RDF/SPARQL/scraping stack (linkeddata, rdf, sparql, shex, mechanize, …) has been removed.
 
 ### Schema Validation
 
@@ -82,7 +91,7 @@ Tests use RSpec with:
 - **VCR** — recorded HTTP cassettes in `spec/vcr_cassettes/` (7-day re-record interval)
 - **WebMock** — disables external HTTP in tests
 
-Test fixtures live in `spec/fixtures/` (YAML, XML, RDF files).
+Test fixtures live in `spec/fixtures/` (YAML and XML files).
 
 ## Style
 

data/Gemfile

@@ -9,6 +9,7 @@ gem "rspec", "~> 3.0"
 
 gem "equivalent-xml"
 gem "ruby-jing"
+gem "rubyzip" # test-only: spec/support reads a zipped index fixture
 gem "simplecov"
 gem "vcr"
 gem "webmock"

data/lib/relaton/w3c/data_fetcher.rb

@@ -1,14 +1,28 @@
 require "relaton/core"
 require "w3c_api"
 require_relative "../w3c"
-require_relative "rate_limit_handler"
+require_relative "safe_realize"
 require_relative "data_parser"
 require_relative "pubid"
 
 module Relaton
   module W3c
     class DataFetcher < Core::DataFetcher
-      include Relaton::W3c::RateLimitHandler
+      include Relaton::W3c::SafeRealize
+
+      DEFAULT_CONCURRENCY = 8
+
+      # Number of fetch_spec worker threads. Tunable via env var so CI or
+      # local runs can dial it down (e.g. for debugging or to lighten load
+      # on api.w3.org).
+      def self.concurrency
+        (ENV["RELATON_W3C_FETCH_CONCURRENCY"] || DEFAULT_CONCURRENCY).to_i
+      end
+
+      def initialize(*args)
+        super
+        @mutex = Mutex.new
+      end
 
       def index
         @index ||= Relaton::Index.find_or_create(:W3C, file: "#{INDEXFILE}.yaml")
@@ -23,41 +37,59 @@ module Relaton
       end
 
       #
-      # Parse documents
+      # Parse documents in parallel. The crawler is heavily I/O-bound on
+      # api.w3.org round-trips (~30-50k requests per run), so a small thread
+      # pool gives a near-linear speedup. Pagination still happens serially
+      # because each page depends on the previous response's `next` link.
       #
       def fetch(_source = nil)
+        n_workers = self.class.concurrency
+        queue = SizedQueue.new(n_workers * 4)
+        workers = Array.new(n_workers) { spawn_worker(queue) }
+
         specs = client.specifications
         loop do
-          specs.links.specifications.each do |spec|
-            fetch_spec spec
-          end
-
+          specs.links.specifications.each { |spec| queue << spec }
           break unless specs.next?
 
-          specs = specs.next
+          # Route pagination through realize so transient 403/5xx on the
+          # next-page link retry with backoff instead of crashing the crawl.
+          next_page = realize(specs.links.next)
+          break unless next_page
+
+          specs = next_page
         end
+
+        n_workers.times { queue << nil } # poison pills
+        workers.each(&:join)
+
         index.save
         report_errors
       end
 
       def fetch_spec(unrealized_spec)
         spec = realize unrealized_spec
-        save_doc DataParser.parse(spec, @errors)
+        return unless spec
+
+        local_errors = Hash.new(true)
+        save_doc DataParser.parse(spec, local_errors)
 
         if spec.links.respond_to?(:version_history) && spec.links.version_history
           version_history = realize spec.links.version_history
-          version_history.links.spec_versions.each { |version| save_doc DataParser.parse(realize version) }
+          version_history&.links&.spec_versions&.each { |version| parse_and_save version }
         end
 
         if spec.links.respond_to?(:predecessor_versions) && spec.links.predecessor_versions
           predecessor_versions = realize spec.links.predecessor_versions
-          predecessor_versions.links.predecessor_versions.each { |version| save_doc DataParser.parse(realize version) }
+          predecessor_versions&.links&.predecessor_versions&.each { |version| parse_and_save version }
         end
 
         if spec.links.respond_to?(:successor_versions) && spec.links.successor_versions
           successor_versions = realize spec.links.successor_versions
-          successor_versions.links.successor_versions.each { |version| save_doc DataParser.parse(realize version) }
+          successor_versions&.links&.successor_versions&.each { |version| parse_and_save version }
         end
+
+        @mutex.synchronize { local_errors.each { |k, v| @errors[k] &&= v } }
       end
 
       #
@@ -69,14 +101,16 @@ module Relaton
         return unless bib
 
         file = file_name(bib.docnumber)
-        if @files.include?(file)
-          Util.warn "File #{file} already exists. Document: #{bib.docnumber}" if warn_duplicate
-        else
-          pubid = PubId.parse bib.docnumber
-          index.add_or_update pubid.to_hash, file
-          @files << file
+        @mutex.synchronize do
+          if @files.include?(file)
+            Util.warn "File #{file} already exists. Document: #{bib.docnumber}" if warn_duplicate
+          else
+            pubid = PubId.parse bib.docnumber
+            index.add_or_update pubid.to_hash, file
+            @files << file
+          end
+          File.write file, serialize(bib), encoding: "UTF-8"
         end
-        File.write file, serialize(bib), encoding: "UTF-8"
       end
 
       def to_xml(bib)
@@ -102,6 +136,26 @@ module Relaton
         name = id.sub(/^W3C\s/, "").gsub(/[\s,:\/+]/, "_").squeeze("_").downcase
         File.join @output, "#{name}.#{@ext}"
       end
+
+      private
+
+      def spawn_worker(queue)
+        Thread.new do
+          while (spec = queue.pop)
+            begin
+              fetch_spec spec
+            rescue StandardError => e
+              log_error "fetch_spec failed: #{e.class}: #{e.message}\n" \
+                        "#{e.backtrace.first(5).join("\n")}"
+            end
+          end
+        end
+      end
+
+      def parse_and_save(version)
+        realized = realize version
+        save_doc DataParser.parse(realized) if realized
+      end
     end
   end
 end

data/lib/relaton/w3c/data_parser.rb

@@ -1,7 +1,7 @@
 module Relaton
   module W3c
     class DataParser
-      include Relaton::W3c::RateLimitHandler
+      include Relaton::W3c::SafeRealize
 
       USED_TYPES = %w[WD NOTE PER PR REC CR].freeze
 
@@ -236,10 +236,11 @@ module Relaton
       def parse_relation
         result = if @spec.links.respond_to?(:version_history)
                    version_history = realize @spec.links.version_history
-                   version_history.links.spec_versions.map { |version| create_relation(version, "hasEdition") }
+                   version_history&.links&.spec_versions&.map { |version| create_relation(version, "hasEdition") } || []
                  else
                    relations
                  end
+        result = result.compact
         @errors[:relation] &&= result.empty?
         result
       end
@@ -254,17 +255,17 @@ module Relaton
         rels << create_relation(@spec.links.specification, "editionOf") if @spec.links.respond_to?(:specification)
         if @spec.links.respond_to?(:predecessor_versions) && @spec.links.predecessor_versions
           predecessor_versions = realize @spec.links.predecessor_versions
-          predecessor_versions.links.predecessor_versions.each do |version|
+          predecessor_versions&.links&.predecessor_versions&.each do |version|
             rels << create_relation(version, "obsoletes")
           end
         end
         if @spec.links.respond_to?(:successor_versions) && @spec.links.successor_versions
           successor_versions = realize @spec.links.successor_versions
-          successor_versions.links.successor_versions.each do |version|
+          successor_versions&.links&.successor_versions&.each do |version|
             rels << create_relation(version, "updatedBy", "errata")
           end
         end
-        rels
+        rels.compact
       end
 
       #
@@ -278,6 +279,8 @@ module Relaton
       #
       def create_relation(version, type, desc = nil)
         version_spec = realize version
+        return nil unless version_spec
+
         url = doc_uri(version_spec)
         id = pub_id(url)
         title = parse_title(version_spec)
@@ -314,7 +317,7 @@ module Relaton
 
         if @spec.links.respond_to?(:editors)
           editors = realize @spec.links.editors
-          editors.links.editors&.each do |ed|
+          editors&.links&.editors&.each do |ed|
             editor = create_editor(ed)
             contribs << editor if editor
           end

data/lib/relaton/w3c/safe_realize.rb

@@ -0,0 +1,55 @@
+require "concurrent/map"
+
+module Relaton
+  module W3c
+    # Thin wrapper over lutaml-hal's `realize`. Successful objects are cached by
+    # w3c_api (it caches realized objects keyed by URL), so this only remembers
+    # resources that failed terminally and returns nil for them — so one broken
+    # link doesn't abort the crawl and isn't re-fetched on every reference.
+    #
+    # Transient failures are retried upstream: w3c_api retries HTTP 403 (the
+    # W3C rate-limit signal) and connection/timeout errors, and lutaml-hal
+    # retries 429 and 5xx. By the time an error surfaces here it is terminal.
+    module SafeRealize
+      # Hrefs that failed terminally — one map shared by every includer
+      # (DataFetcher and DataParser) since a broken resource is broken for the
+      # whole crawl. Initialized eagerly (at load, single-threaded) so the
+      # parallel fetcher's first concurrent access can't race a lazy `||=`;
+      # Concurrent::Map then handles the concurrent reads/writes.
+      @skipped = Concurrent::Map.new
+
+      def self.skipped
+        @skipped
+      end
+
+      def realize(obj)
+        href = resolve_href(obj)
+        return nil if SafeRealize.skipped.key?(href)
+
+        obj.realize
+      rescue Lutaml::Hal::ConnectionError, Lutaml::Hal::TimeoutError, Faraday::Error, Net::OpenTimeout => e
+        # Network-level failure (already retried by w3c_api). The resource itself
+        # is fine, so don't skip it permanently — a later reference can try again.
+        Util.warn "Failed to realize object: #{href}, error: #{e.message}"
+        nil
+      rescue Lutaml::Hal::NotFoundError
+        Util.warn "Object not found: #{href}"
+        SafeRealize.skipped[href] = true
+        nil
+      rescue Lutaml::Hal::Error => e
+        # Definitive upstream error (403 rate-limit, 5xx, 429) already retried by
+        # w3c_api / lutaml-hal. Skip the broken/unavailable resource rather than
+        # re-hitting it for every link that references it.
+        Util.warn "Skipping #{href}, upstream error after retries: #{e.message}"
+        SafeRealize.skipped[href] = true
+        nil
+      end
+
+      private
+
+      def resolve_href(obj)
+        obj.href || obj.links.self.href
+      end
+    end
+  end
+end

data/lib/relaton/w3c/version.rb

@@ -1,5 +1,5 @@
 module Relaton
   module W3c
-    VERSION = "2.1.1".freeze
+    VERSION = "2.1.3".freeze
   end
 end

data/relaton-w3c.gemspec

@@ -31,16 +31,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "linkeddata", "~> 3.2"
-  spec.add_dependency "mechanize", "~> 2.10"
-  spec.add_dependency "rdf", "~> 3.2"
-  spec.add_dependency "rdf-normalize", "~> 0.6"
   spec.add_dependency "relaton-bib", "~> 2.1.0"
   spec.add_dependency "relaton-core", "~> 0.0.13"
   spec.add_dependency "relaton-index", "~> 0.2.8"
-  spec.add_dependency "rubyzip", "~> 2.3"
-  spec.add_dependency "shex", "~> 0.7"
-  spec.add_dependency "csv", "~> 3.0"
-  spec.add_dependency "sparql", "~> 3.2"
-  spec.add_dependency "w3c_api", "~> 0.1.3"
+  spec.add_dependency "w3c_api", "~> 0.3.2"
 end

metadata

@@ -1,71 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: relaton-w3c
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.3
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-05 00:00:00.000000000 Z
+date: 2026-06-03 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: linkeddata
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-- !ruby/object:Gem::Dependency
-  name: mechanize
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.10'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.10'
-- !ruby/object:Gem::Dependency
-  name: rdf
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-- !ruby/object:Gem::Dependency
-  name: rdf-normalize
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
@@ -108,76 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.8
-- !ruby/object:Gem::Dependency
-  name: rubyzip
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.3'
-- !ruby/object:Gem::Dependency
-  name: shex
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.7'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.7'
-- !ruby/object:Gem::Dependency
-  name: csv
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: sparql
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: w3c_api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.3.2
 description: 'Relaton::W3c: retrieve W3C Standards for bibliographic using the IsoBibliographicItem
   model'
 email:
@@ -200,11 +88,6 @@ files:
 - bin/console
 - bin/rspec
 - bin/setup
-- grammars/basicdoc.rng
-- grammars/biblio-standoc.rng
-- grammars/biblio.rng
-- grammars/relaton-w3c-compile.rng
-- grammars/relaton-w3c.rng
 - lib/relaton/w3c.rb
 - lib/relaton/w3c/bibdata.rb
 - lib/relaton/w3c/bibitem.rb
@@ -217,7 +100,7 @@ files:
 - lib/relaton/w3c/item_data.rb
 - lib/relaton/w3c/processor.rb
 - lib/relaton/w3c/pubid.rb
-- lib/relaton/w3c/rate_limit_handler.rb
+- lib/relaton/w3c/safe_realize.rb
 - lib/relaton/w3c/util.rb
 - lib/relaton/w3c/version.rb
 - relaton-w3c.gemspec