relaton-w3c 2.1.1 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36c34b0daf914dbc2cf41cd4fe4842fc04d67147d977ac1282b9d7f0452538d7
-  data.tar.gz: f6f0fdf9774a4bdb798b4d00e0ba4dae18e6d9ec772ce80d6604d5666fb22893
+  metadata.gz: 46f27195b30d4b285034731e6749b4ed4aff9ed607d41530a1d5b9d07c859b22
+  data.tar.gz: a263250dc4bda6a871af9addecbe37c5e3d62ab748797fe064dc9545e90eec8a
 SHA512:
-  metadata.gz: a6b3105b9b30556d46badf314f8c298718422c2cda7b9e015af0469cf64d99205238c7e786b1d7d429277ef2f58498241f99a2f170463ff273f94a53e1d58857
-  data.tar.gz: f51e0ee34082f78e51d0da502087ecd4ca7b7cb4e1903f7b2db20123ebddaf56d57c655c057825022df459f02e50ade7b1cbb7d3ad34aff4c621240aeebe6006
+  metadata.gz: 973bcf91864d27cb1f19f6001733dc0b46c66738d1e6614ff91313a61dcf03b6a721cbc8c82fb3bd64a00dccc8af50dd2efbf23d051ada503d4b8c698719bb68
+  data.tar.gz: 89fb5bcce023f69488932d901fbe260450bf1fcd6164c46be7731fb107bcb47a9aec30b9eb08a7fe534822727dcdb48c262e2386cf7f921ce018f0537fb33377

data/lib/relaton/w3c/data_fetcher.rb

@@ -10,6 +10,20 @@ module Relaton
     class DataFetcher < Core::DataFetcher
       include Relaton::W3c::RateLimitHandler
 
+      DEFAULT_CONCURRENCY = 8
+
+      # Number of fetch_spec worker threads. Tunable via env var so CI or
+      # local runs can dial it down (e.g. for debugging or to lighten load
+      # on api.w3.org).
+      def self.concurrency
+        (ENV["RELATON_W3C_FETCH_CONCURRENCY"] || DEFAULT_CONCURRENCY).to_i
+      end
+
+      def initialize(*args)
+        super
+        @mutex = Mutex.new
+      end
+
       def index
         @index ||= Relaton::Index.find_or_create(:W3C, file: "#{INDEXFILE}.yaml")
       end
@@ -23,41 +37,54 @@ module Relaton
       end
 
       #
-      # Parse documents
+      # Parse documents in parallel. The crawler is heavily I/O-bound on
+      # api.w3.org round-trips (~30-50k requests per run), so a small thread
+      # pool gives a near-linear speedup. Pagination still happens serially
+      # because each page depends on the previous response's `next` link.
       #
       def fetch(_source = nil)
+        n_workers = self.class.concurrency
+        queue = SizedQueue.new(n_workers * 4)
+        workers = Array.new(n_workers) { spawn_worker(queue) }
+
         specs = client.specifications
         loop do
-          specs.links.specifications.each do |spec|
-            fetch_spec spec
-          end
-
+          specs.links.specifications.each { |spec| queue << spec }
           break unless specs.next?
 
           specs = specs.next
         end
+
+        n_workers.times { queue << nil } # poison pills
+        workers.each(&:join)
+
         index.save
         report_errors
       end
 
       def fetch_spec(unrealized_spec)
         spec = realize unrealized_spec
-        save_doc DataParser.parse(spec, @errors)
+        return unless spec
+
+        local_errors = Hash.new(true)
+        save_doc DataParser.parse(spec, local_errors)
 
         if spec.links.respond_to?(:version_history) && spec.links.version_history
           version_history = realize spec.links.version_history
-          version_history.links.spec_versions.each { |version| save_doc DataParser.parse(realize version) }
+          version_history&.links&.spec_versions&.each { |version| parse_and_save version }
         end
 
         if spec.links.respond_to?(:predecessor_versions) && spec.links.predecessor_versions
           predecessor_versions = realize spec.links.predecessor_versions
-          predecessor_versions.links.predecessor_versions.each { |version| save_doc DataParser.parse(realize version) }
+          predecessor_versions&.links&.predecessor_versions&.each { |version| parse_and_save version }
         end
 
         if spec.links.respond_to?(:successor_versions) && spec.links.successor_versions
           successor_versions = realize spec.links.successor_versions
-          successor_versions.links.successor_versions.each { |version| save_doc DataParser.parse(realize version) }
+          successor_versions&.links&.successor_versions&.each { |version| parse_and_save version }
         end
+
+        @mutex.synchronize { local_errors.each { |k, v| @errors[k] &&= v } }
       end
 
       #
@@ -69,14 +96,16 @@ module Relaton
         return unless bib
 
         file = file_name(bib.docnumber)
-        if @files.include?(file)
-          Util.warn "File #{file} already exists. Document: #{bib.docnumber}" if warn_duplicate
-        else
-          pubid = PubId.parse bib.docnumber
-          index.add_or_update pubid.to_hash, file
-          @files << file
+        @mutex.synchronize do
+          if @files.include?(file)
+            Util.warn "File #{file} already exists. Document: #{bib.docnumber}" if warn_duplicate
+          else
+            pubid = PubId.parse bib.docnumber
+            index.add_or_update pubid.to_hash, file
+            @files << file
+          end
+          File.write file, serialize(bib), encoding: "UTF-8"
         end
-        File.write file, serialize(bib), encoding: "UTF-8"
       end
 
       def to_xml(bib)
@@ -102,6 +131,26 @@ module Relaton
         name = id.sub(/^W3C\s/, "").gsub(/[\s,:\/+]/, "_").squeeze("_").downcase
         File.join @output, "#{name}.#{@ext}"
       end
+
+      private
+
+      def spawn_worker(queue)
+        Thread.new do
+          while (spec = queue.pop)
+            begin
+              fetch_spec spec
+            rescue StandardError => e
+              log_error "fetch_spec failed: #{e.class}: #{e.message}\n" \
+                        "#{e.backtrace.first(5).join("\n")}"
+            end
+          end
+        end
+      end
+
+      def parse_and_save(version)
+        realized = realize version
+        save_doc DataParser.parse(realized) if realized
+      end
     end
   end
 end

data/lib/relaton/w3c/data_parser.rb

@@ -236,10 +236,11 @@ module Relaton
       def parse_relation
         result = if @spec.links.respond_to?(:version_history)
                    version_history = realize @spec.links.version_history
-                   version_history.links.spec_versions.map { |version| create_relation(version, "hasEdition") }
+                   version_history&.links&.spec_versions&.map { |version| create_relation(version, "hasEdition") } || []
                  else
                    relations
                  end
+        result = result.compact
         @errors[:relation] &&= result.empty?
         result
       end
@@ -254,17 +255,17 @@ module Relaton
         rels << create_relation(@spec.links.specification, "editionOf") if @spec.links.respond_to?(:specification)
         if @spec.links.respond_to?(:predecessor_versions) && @spec.links.predecessor_versions
           predecessor_versions = realize @spec.links.predecessor_versions
-          predecessor_versions.links.predecessor_versions.each do |version|
+          predecessor_versions&.links&.predecessor_versions&.each do |version|
             rels << create_relation(version, "obsoletes")
           end
         end
         if @spec.links.respond_to?(:successor_versions) && @spec.links.successor_versions
           successor_versions = realize @spec.links.successor_versions
-          successor_versions.links.successor_versions.each do |version|
+          successor_versions&.links&.successor_versions&.each do |version|
             rels << create_relation(version, "updatedBy", "errata")
           end
         end
-        rels
+        rels.compact
       end
 
       #
@@ -278,6 +279,8 @@ module Relaton
       #
       def create_relation(version, type, desc = nil)
         version_spec = realize version
+        return nil unless version_spec
+
         url = doc_uri(version_spec)
         id = pub_id(url)
         title = parse_title(version_spec)
@@ -314,7 +317,7 @@ module Relaton
 
         if @spec.links.respond_to?(:editors)
           editors = realize @spec.links.editors
-          editors.links.editors&.each do |ed|
+          editors&.links&.editors&.each do |ed|
             editor = create_editor(ed)
             contribs << editor if editor
           end

data/lib/relaton/w3c/rate_limit_handler.rb

@@ -1,3 +1,5 @@
+require "concurrent/map"
+
 module Relaton
   module W3c
     module RateLimitHandler
@@ -7,8 +9,11 @@ module Relaton
         Lutaml::Hal::ServerError, Faraday::ConnectionFailed, Net::OpenTimeout,
       ].freeze
 
+      # Concurrent::Map so multiple fetcher threads can hit the cache without
+      # a global lock. Duplicate concurrent fetches of the same URL are
+      # possible but harmless; the second write just replaces the first.
       def self.fetched_objects
-        @fetched_objects ||= {}
+        @fetched_objects ||= Concurrent::Map.new
       end
 
       def realize(obj)
@@ -38,6 +43,12 @@ module Relaton
         rescue Lutaml::Hal::NotFoundError
           Util.warn "Object not found: #{href}"
           RateLimitHandler.fetched_objects[href] = nil
+        rescue Lutaml::Hal::Error => e
+          # Non-retryable client-side errors (403/401/400 and any other
+          # Lutaml::Hal::Error not matched above) — skip the resource and
+          # continue rather than aborting the whole crawl.
+          Util.warn "Client error for #{href}, skipping: #{e.message}"
+          RateLimitHandler.fetched_objects[href] = nil
         end
       end
 

data/lib/relaton/w3c/version.rb

@@ -1,5 +1,5 @@
 module Relaton
   module W3c
-    VERSION = "2.1.1".freeze
+    VERSION = "2.1.2".freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-w3c
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-05 00:00:00.000000000 Z
+date: 2026-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: linkeddata