relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/core/hit.rb

@@ -0,0 +1,49 @@
+require "weakref"
+
+module Relaton
+  module Core
+    class Hit
+      # @return [RelatonBib::HitCollection]
+      attr_accessor :hit_collection
+
+      # @return [Array<Hash>]
+      attr_reader :hit
+
+      # @param hit [Hash]
+      # @param hit_collection [RelatonBib::HitCollection]
+      def initialize(hit, hit_collection = nil)
+        @hit            = hit
+        @hit_collection = WeakRef.new hit_collection if hit_collection
+      end
+
+      # @return [String]
+      def to_s
+        inspect
+      end
+
+      # @return [String]
+      def inspect
+        "<#{self.class}:#{format('%<id>#.14x', id: object_id << 1)} " \
+          "@reference=\"#{@hit_collection&.ref}\" " \
+          "@fetched=\"#{fetched?}\" " \
+          "@docidentifier=\"#{@hit[:code]}\">"
+      end
+
+      # @return [RelatonBib::ItemData]
+      def item
+        raise "Not implemented"
+      end
+
+      def fetched?
+        !@item.nil?
+      end
+
+      # @param opts [Hash]
+      # @option opts [Boolean] :bibdata
+      # @return [String] XML
+      def to_xml(**opts)
+        item.to_xml(**opts)
+      end
+    end
+  end
+end

data/lib/relaton/core/hit_collection.rb

@@ -0,0 +1,118 @@
+require "forwardable"
+require "nokogiri"
+require "weakref"
+require_relative "hit"
+
+module Relaton
+  module Core
+    class HitCollection
+      extend Forwardable
+
+      def_delegators :@array, :<<, :[], :first, :empty?, :any?, :size, :each, :each_slice, :reduce, :map
+
+      # @return [TrueClass, FalseClass]
+      attr_reader :fetched
+
+      # @return [String]
+      attr_reader :ref
+
+      # @return [String]
+      attr_reader :year
+
+      #
+      # @param ref [String, Pubid] reference to search
+      # @param year [String, nil] year of publication
+      #
+      def initialize(ref, year = nil)
+        @array = []
+        @ref = ref
+        @year = year
+        @fetched = false
+      end
+
+      #
+      # Fetches hits from the data source
+      #
+      # @return [self] self object
+      #
+      def fetch
+        workers = WorkersPool.new 4
+        workers.worker(&:item)
+        each do |hit|
+          workers << hit
+        end
+        workers.end
+        workers.result
+        @fetched = true
+        self
+      end
+
+      #
+      # Renders the collection as XML
+      #
+      # @param opts [Hash] options
+      # @option opts [Nokogiri::XML::Builder] :builder XML builder
+      # @option opts [Boolean] :bibdata render bibdata if true
+      # @option opts [String, Symbol] :lang language
+      #
+      # @return [String] XML representation of the collection
+      #
+      def to_xml(**opts)
+        builder = Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
+          xml.documents do
+            @array.each do |hit|
+              xml << hit.to_xml(**opts)
+            end
+          end
+        end
+        builder.to_xml
+      end
+
+      #
+      # Selects matching hits and returns a new collection
+      #
+      # @param [Proc] &block proc to select hits
+      #
+      # @return [RelatonBib::HitCollection] new hit collection
+      #
+      def select!(&block)
+        @array.select!(&block)
+        self
+      end
+
+      def select(&block)
+        array = @array.select(&block)
+        self.class.new(@ref, @year).tap do |hc|
+          new_array = array.map do |hit|
+            hit.dup.tap { |h| h.hit_collection = WeakRef.new(hc) }
+          end
+          hc.instance_variable_set(:@array, new_array)
+          hc.instance_variable_set(:@fetched, @fetched)
+        end
+      end
+
+      def reduce!(sum, &block)
+        @array = @array.reduce sum, &block
+        self
+      end
+
+      #
+      # Returns String representation of the collection
+      #
+      # @return [String] String representation of the collection
+      #
+      def to_s
+        inspect
+      end
+
+      #
+      # Returns String representation of the collection
+      #
+      # @return [String] String representation of the collection
+      #
+      def inspect
+        "<#{self.class}:#{format('%#.14x', object_id << 1)} @ref=#{@ref} @fetched=#{@fetched}>"
+      end
+    end
+  end
+end

data/lib/relaton/core/pacer.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Core
+    #
+    # A process-wide, thread-safe request pacer: the whole worker pool between
+    # them starts **at most one request per `gap` seconds**.
+    #
+    # It exists because the two natural alternatives are both wrong for a
+    # crawler that has to be polite to one host:
+    #
+    # * `sleep delay` before each request (what `Relaton::Itu::DataCrawlerR`
+    #   did) **adds** the delay to the server's own latency. At ITU's measured
+    #   ~1.1 s response time a 1 s delay yields one request every ~2.1 s, so a
+    #   ~7k-request crawl spends ~2 h of a ~4 h run doing nothing at all — and
+    #   the crawl is pinned to one thread, because the pacing *is* the loop.
+    # * a per-worker gap (`Relaton::Cie::DataFetcher::Pacing`) lets N workers
+    #   issue N requests per gap. That is right for CIE, where the point is to
+    #   let one struggling worker slow itself without dragging the others down,
+    #   and wrong here, where the budget belongs to the host.
+    #
+    # This reserves a *slot* instead: each caller takes the next free instant on
+    # a shared timeline and sleeps only until it arrives. The server's latency
+    # therefore counts toward the gap rather than being added on top, so the
+    # rate the host sees is exactly `1 / gap` no matter how many workers there
+    # are — and workers exist only to hide latency, never to raise the rate.
+    #
+    # The mutex is held for the reservation and **never across the sleep**, so a
+    # waiting worker cannot stop another from reserving.
+    #
+    # Idle time does not bank up slots: a reservation never starts earlier than
+    # "now", so a stall elsewhere in the crawl (a governor cooldown, a slow
+    # merge) cannot be repaid afterwards as a burst — which is exactly the burst
+    # that would re-trip the limiter that caused the stall.
+    #
+    # `clock:` and `sleeper:` are injectable so specs exercise the arithmetic
+    # without wall-clock waits (mirroring `Relaton::Core::Governor`).
+    #
+    class Pacer
+      # How the gap is spent.
+      #
+      #   :slot   (default) the gap is the minimum interval between request
+      #           *starts*, shared by the pool, so the host's own latency counts
+      #           toward it. Peak rate 1/gap regardless of worker count.
+      #   :fixed  sleep the whole gap before every request, whatever else is
+      #           happening. This is what a plain `sleep delay` loop did, kept
+      #           as a rollback knob: with one worker it reproduces the old
+      #           request pattern exactly — one request per (gap + latency).
+      DEFAULT_MODE = :slot
+
+      # @param gap [Numeric] minimum seconds between request *starts*. Zero
+      #   disables pacing entirely (used by specs and by cassette replay).
+      # @param mode [Symbol] :slot or :fixed, see DEFAULT_MODE
+      # @param clock [#call, nil] returns monotonic seconds
+      # @param sleeper [#call, nil] receives seconds to sleep
+      def initialize(gap:, mode: DEFAULT_MODE, clock: nil, sleeper: nil)
+        @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+        @sleeper = sleeper || ->(secs) { sleep secs }
+        @mutex = Mutex.new
+        @gap = coerce_gap(gap)
+        @mode = mode == :fixed ? :fixed : DEFAULT_MODE
+        @next = nil
+        @requests = 0
+        @slept = 0.0
+      end
+
+      attr_reader :gap, :mode
+
+      # @return [Integer] requests paced so far, for the run summary
+      def requests
+        @mutex.synchronize { @requests }
+      end
+
+      # @return [Float] total seconds spent waiting, for the run summary
+      def slept
+        @mutex.synchronize { @slept }
+      end
+
+      # @param secs [Numeric] the new minimum gap; takes effect on the next
+      #   reservation. A negative value is clamped to zero.
+      def gap=(secs)
+        @mutex.synchronize { @gap = coerce_gap(secs) }
+      end
+
+      # Clear the timeline and the counters, so the next caller goes out
+      # immediately. For specs, and for a crawl starting a fresh phase.
+      def reset!
+        @mutex.synchronize do
+          @next = nil
+          @requests = 0
+          @slept = 0.0
+        end
+      end
+
+      #
+      # Reserve the next slot and block until it arrives.
+      #
+      # @return [Float] seconds actually slept (0.0 when the slot was already
+      #   due, which is the common case once the host is slower than the gap)
+      #
+      def wait
+        delay = @mode == :fixed ? @mutex.synchronize { @gap } : (reserve - @clock.call)
+        @mutex.synchronize { @requests += 1 }
+        return 0.0 unless delay.positive?
+
+        @sleeper.call delay
+        @mutex.synchronize { @slept += delay }
+        delay
+      end
+
+      private
+
+      # Claim the next instant on the shared timeline and advance it. Held under
+      # the lock; the waiting happens outside.
+      #
+      # @return [Float] the instant this caller may start its request
+      def reserve
+        @mutex.synchronize do
+          now = @clock.call
+          # `max(now)` is what stops an idle stretch from banking up slots: the
+          # timeline catches up to the present rather than paying out a burst.
+          @next = @next.nil? || @next < now ? now : @next
+          reserved = @next
+          @next += @gap
+          reserved
+        end
+      end
+
+      def coerce_gap(secs)
+        [secs.to_f, 0.0].max
+      end
+    end
+  end
+end

data/lib/relaton/core/processor.rb

@@ -0,0 +1,67 @@
+module Relaton
+  module Core
+    class Processor
+      # @rerurn [Symbol]
+      attr_reader :short
+
+      # @return [String]
+      attr_reader :prefix, :idtype
+
+      # @return [Regexp]
+      attr_reader :defaultprefix
+
+      # @return [Array<String>]
+      attr_reader :datasets
+
+      # Global document-ID prefixes this flavor owns, e.g. BSI => %w[BS BSI DD …].
+      # The source of truth is **pubid**: a flavor sets @pubid_flavor to its Pubid
+      # module name (e.g. :Iso) in #initialize, and the prefixes are read from
+      # `Pubid::<Flavor>.prefixes` — the SDO's own leading identifier tokens,
+      # including non-obvious ones (BSI `DD`) and joint forms (`ISO/IEC`). Flavors
+      # with no pubid backing fall back to the single canonical @prefix. Loaded
+      # lazily (pubid is only required on first call) and memoized. Feeds the
+      # global prefix register (Relaton.prefix_flavor). (relaton-db#103)
+      #
+      # @return [Array<String>]
+      def prefixes
+        @prefixes ||= if @pubid_flavor
+                        require "pubid"
+                        ::Pubid.const_get(@pubid_flavor).prefixes
+                      else
+                        Array(prefix)
+                      end
+      end
+
+      def initialize
+        raise "This is an abstract class!"
+      end
+
+      def get(_code, _date, _opts)
+        raise "This is an abstract class!"
+      end
+
+      def fetch_data(_source, _opts)
+        raise "This is an abstract class!"
+      end
+
+      def from_xml(_xml)
+        raise "This is an abstract class!"
+      end
+
+      def from_yaml(_hash)
+        raise "This is an abstract class!"
+      end
+
+      def grammar_hash
+        raise "This is an abstract class!"
+      end
+
+      # Retuns default number of workers. Should be overraded by childred classes if need.
+      #
+      # @return [Integer] nuber of wokrers
+      def threads
+        10
+      end
+    end
+  end
+end

data/lib/relaton/core/request_error.rb

@@ -0,0 +1,14 @@
+module Relaton
+  # Transport failure while fetching remote data; `Relaton::Db#net_retry`
+  # retries it. It depends on nothing, so `relaton/index` requires this file
+  # alone and does not load the rest of `relaton/core`.
+  #
+  # The class is named `Relaton::RequestError`, not `Relaton::Core::...`: an
+  # error raised without a message uses the class name as its message, and
+  # relaton-cli prints that message.
+  class RequestError < StandardError; end
+
+  module Core
+    RequestError = Relaton::RequestError
+  end
+end

data/lib/relaton/core/workers_pool.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Core
+    # Workers poll.
+    class WorkersPool
+      attr_accessor :nb_hits
+
+      def initialize(num_workers = 2)
+        @num_workers = num_workers < 2 ? 2 : num_workers
+        @queue = SizedQueue.new(num_workers * 2)
+        @result = []
+        @nb_hits = 0
+      end
+
+      def worker(&block)
+        @threads = Array.new @num_workers do
+          Thread.new do
+            until (item = @queue.pop) == :END
+              @result << yield(item) if block
+            end
+          end
+        end
+      end
+
+      def result
+        @threads.each(&:join)
+        @result
+      end
+
+      def <<(item)
+        @queue << item
+        self
+      end
+
+      def end
+        @num_workers.times { @queue << :END }
+      end
+
+      def size
+        @result.size
+      end
+    end
+  end
+end

data/lib/relaton/core.rb

@@ -0,0 +1,12 @@
+require "relaton/logger"
+require_relative "version"
+require_relative "core/request_error"
+require_relative "core/array_wrapper"
+require_relative "core/hash_keys_sybolizer"
+require_relative "core/date_parser"
+require_relative "core/processor"
+require_relative "core/data_fetcher"
+require_relative "core/hit_collection"
+require_relative "core/workers_pool"
+require_relative "core/pacer"
+require_relative "core/governor"

data/lib/relaton/db/registry.rb

@@ -11,6 +11,8 @@ module Relaton
         relaton/bsi relaton/cen relaton/iana relaton/3gpp
         relaton/oasis relaton/doi relaton/jis relaton/xsf
         relaton/ccsds relaton/etsi relaton/isbn relaton/plateau
+        relaton/oiml relaton/jcgm relaton/easc relaton/gost relaton/adobe
+        relaton/iala
       ].freeze
 
       include Singleton
@@ -24,10 +26,9 @@ module Relaton
 
       def register_gems
         SUPPORTED_GEMS.each do |b|
-          # Require the gem's top-level file so its internal modules
-          # (e.g. Relaton::Iso::Util) load in the right order before we
-          # poke at its Processor class.
-          require b
+          # Load ONLY the lightweight processor file; its heavy flavor code is
+          # lazy-required inside the processor's methods. This keeps
+          # `require "relaton"` from eagerly loading every flavor at start.
           require "#{b}/processor"
           register Kernel.const_get "#{gem_to_module_path(b)}::Processor"
         rescue LoadError => e
@@ -107,8 +108,47 @@ module Relaton
         nil
       end
 
+      #
+      # Global prefix register (relaton-db#103): all processors that own the
+      # given global prefix, matched case-insensitively and exactly (unlike
+      # #class_by_ref, which matches a prefix at the start of a full reference).
+      # Ordered by registration order (SUPPORTED_GEMS), so results are
+      # deterministic. Lazy: never dereferences a flavor constant.
+      #
+      # @param prefix [String]
+      # @return [Array<Relaton::Core::Processor>]
+      #
+      def processors_by_prefix(prefix)
+        key = prefix.to_s.strip.upcase
+        processors.values.select do |processor|
+          processor.prefixes.any? { |pref| pref.upcase == key }
+        end
+      end
+
+      #
+      # Flavor modules (e.g. Relaton::Iso) that own the given global prefix.
+      # NOTE: dereferencing the returned module forces that flavor's lazy load.
+      # Callers that must stay lazy should use #processors_by_prefix instead.
+      #
+      # @param prefix [String]
+      # @return [Array<Module>]
+      #
+      def flavors_by_prefix(prefix)
+        processors_by_prefix(prefix).map { |processor| flavor_module(processor) }
+      end
+
       private
 
+      # The flavor namespace for a processor: the module enclosing its class.
+      # Relaton::Iso::Processor -> Relaton::Iso. Derived from the class name so
+      # there's no @short string-munging and no 3gpp special case.
+      #
+      # @param processor [Relaton::Core::Processor]
+      # @return [Module]
+      def flavor_module(processor)
+        Object.const_get processor.class.name.split("::")[0..-2].join("::")
+      end
+
       def gem_to_module_path(gem_name)
         gem_name.split("/").map do |part|
           part.capitalize.sub("3gpp", "ThreeGpp")

data/lib/relaton/db.rb

@@ -462,7 +462,6 @@ module Relaton
   end
 end
 
-require_relative "db/version"
 require_relative "db/util"
 require_relative "db/config"
 require_relative "db/workers_pool"

data/lib/relaton/doi/crossref.rb

@@ -0,0 +1,89 @@
+require "mechanize"
+
+module Relaton
+  module Doi
+    module Crossref
+      extend self
+
+      USER_AGENT = "Relaton::Doi (https://www.relaton.org/guides/doi/; mailto:open.source@ribose.com)"
+
+      #
+      # Get a document by DOI from the CrossRef API.
+      #
+      # @param [String] doi The DOI.
+      #
+      # @return [RelatonBib::BibliographicItem, RelatonIetf::IetfBibliographicItem,
+      #   RelatonBipm::BipmBibliographicItem, RelatonIeee::IeeeBibliographicItem,
+      #   RelatonNist::NistBibliographicItem] The bibitem.
+      #
+      def get(doi)
+        Util.info "Fetching from search.crossref.org ...", key: doi
+        id = doi.sub(%r{^doi:}, "")
+        message = get_by_id id
+        if message
+          Util.info "Found: `#{message['DOI']}`", key: doi
+          Parser.parse message
+        else
+          Util.info "Not found.", key: doi
+          nil
+        end
+      end
+
+      #
+      # Get a document by DOI from the CrossRef API.
+      #
+      # @param [String] id The DOI.
+      #
+      # @return [Hash] The document.
+      #
+      def get_by_id(id) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        n = 0
+        url = "https://api.crossref.org/works/#{CGI.escape(id)}"
+        loop do
+          resp = agent.get url
+          work = JSON.parse resp.body
+          return work["message"] if work["status"] == "ok"
+
+          if n > 1
+            raise Relaton::RequestError, "Crossref error: #{resp.body}"
+          end
+
+          n += 1
+          sleep backoff(resp.response, n)
+        rescue Mechanize::ResponseCodeError => e
+          return nil if e.response_code == "404"
+
+          if n > 1
+            raise Relaton::RequestError, "Crossref error: #{e.page.body}"
+          end
+
+          n += 1
+          sleep backoff(e.page.response, n)
+        end
+      end
+
+      #
+      # Seconds to wait before retry n. Crossref sends X-Rate-Limit-Interval as
+      # "1s", but a throttled response can carry no rate-limit headers at all —
+      # the 429s seen here had only Date/Content-Length/Connection. Without the
+      # floor that degenerates to `sleep 0`, i.e. hammering the endpoint that
+      # just asked us to slow down.
+      #
+      # @param [Hash, nil] headers The response headers.
+      # @param [Integer] num The attempt number.
+      #
+      # @return [Integer] Delay in seconds, at least 1.
+      #
+      def backoff(headers, num)
+        interval = headers.to_h["x-rate-limit-interval"].to_s[/\d+/].to_i
+        [interval, 1].max * num
+      end
+
+      def agent
+        @agent ||= Mechanize.new do |a|
+          a.user_agent = USER_AGENT
+        end
+      end
+    end
+  end
+end