relaton 1.7.3 → 1.7.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f957d5a962fa3529fb21f4289ff4dc648029497ec3bf209e084b85c49a336531
-  data.tar.gz: 6e6ad879c1552c8396c8f365c4bdc9c088197424f695b68b0dbeec9cc272f1a4
+  metadata.gz: d0ff67d3ebeb9332a6aa18ad88e8b585478c29adb27ccda935ab9b8af4b34e2b
+  data.tar.gz: 9aefca4601eb5bd7c4491dafc41fcee87cbb7e5d7508ee05086cfa39122bfbef
 SHA512:
-  metadata.gz: f87f3d11aae3e805fa2fe3d9875692c2004a7b7a51fc1235181f8a4a9b314cf0403e50a4134fb093edc0da2cdb42339a7fe4ee7024e25de92d7923bf5029ab35
-  data.tar.gz: 2684a909260936e26f9a2b987c667e61d24e96c01f326bc008deaf9358f1dc349f216c239932ee5780325057bfb192426173d1e226c031880743ff4ce0ad4d12
+  metadata.gz: b69bf1fbbbc4258bd22304dda3d54b23bc8a8e2361aeff2f26487518236e6b46690d0a793dd076489749ae0dcd93a6c3da7bc1969d3e877149ad9722b9ed0e9e
+  data.tar.gz: '0682111008bcb648773cd0e90bb4f5a789007b5d5deca6aa2c5aa245b1f677a136eff936c6256b8e6bf76bcf4cb65f8371ba0cb5fe0c9ffe908e7d4640031342'

data/.github/workflows/rake.yml

@@ -0,0 +1,46 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
+name: rake
+
+on:
+  push:
+    branches: [ master, main ]
+    tags: [ v* ]
+  pull_request:
+
+jobs:
+  rake:
+    name: Test on Ruby ${{ matrix.ruby }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        experimental: [ false ]
+        include:
+          - ruby: '3.0'
+            os: 'ubuntu-latest'
+            experimental: true
+          - ruby: '3.0'
+            os: 'windows-latest'
+            experimental: true
+          - ruby: '3.0'
+            os: 'macos-latest'
+            experimental: true
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      # https://github.com/ruby-debug/debase/issues/89#issuecomment-686827382
+      - if: matrix.os == 'macos-latest' && matrix.ruby == '2.5'
+        run: echo BUNDLE_BUILD__DEBASE="--with-cflags=\"-Wno-error=implicit-function-declaration\"" >> $GITHUB_ENV
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - run: bundle exec rake

data/docs/README.adoc

@@ -64,6 +64,11 @@ e.g. `get("ISO 19115-1", "2014", all_parts: true)` is transformed into a referen
 
 === Create DB
 
+`Relaton::Db#new(globalcache, localcache)` creates new DB. Returns Relaton::Db instance.
+
+* `globalcache` - (String or nil) path to globalcache directory
+* `localcache` - (String or nil) path to localcache directory
+
 [source,ruby]
 ----
 require "relaton"
@@ -97,9 +102,46 @@ db = Relaton::Db.new("globalcache", "localcache")
 ...
 ----
 
+=== Modify DB
+
+==== Move DB
+
+`Relaton::Db#mv(new_dir, type: :global)` moves DB directory to new location. Returns path to new directory if successful, or `nil` if target directiory exists.
+
+* `new_dir` - (String) new cache location
+* `type` - (Symbol) type of cache DB. Allowed values are: `:global`, `:local`. Default is `:global`.
+
+[source,ruby]
+----
+db.mv("new_global_dir")
+=> "new_global_dir"
+
+db.mv("new_local_dir", type: :local)
+=> "new_local_dir"
+----
+
+==== Clear DB
+
+`Relaton::Db#clear` removes all entries form DB
+
 === Fetch documens
 
-==== Fetch by references
+==== Fetch document by references
+
+There are 3 fetching methods:
+
+* `Relaton::Db#fetch(reference, year, options)` - fetches document from local cache or remote source.
+* `Relaton::Db#fetch_db(reference, year, options)` - fetches document from local cache
+* `Relaton::Db#fetch_async(reference, year, options, &block)` - fetches document asynchronously
+
+Arguments:
+
+* `reference` - (String) reference to fethc document
+* `year` - (String or nil) year to filter relult (optional)
+* `options` - (Hash) hash of options. Alloved options:
+- `:all_parts` - (Boolean) should be `true` if all-parts reference is required
+- `:keep_yer` - (Boolean) should be `true` if undated reference should return actual reference with year
+- `:retries` - (Number) number of network retries. Default 1
 
 [source,ruby]
 ----
@@ -114,7 +156,7 @@ x = db.fetch("ISO 19011")
 => #<RelatonIsoBib::IsoBibliographicItem:0x007fb1d0ab2f00
 ...
 
-x = db.fetch("ISO 19011", "2011")
+x = db.fetch("ISO 19011", "2011", retries: 3)
 [relaton-iso] ("ISO 19011") fetching...
 [relaton-iso] ("ISO 19011") found ISO 19011:2011
 => #<RelatonIsoBib::IsoBibliographicItem:0x007fb1d2593068
@@ -125,6 +167,36 @@ x = db.fetch("ISO 19115", nil, all_parts: true)
 [relaton-iso] ("ISO 19115") found ISO 19115 (all parts)
 => #<RelatonIsoBib::IsoBibliographicItem:0x007fb1d0ae8bf0
 ...
+
+# Fetchig from local cache
+
+x = db.fetch("ISO 19011")
+=> #<RelatonIsoBib::IsoBibliographicItem:0x007fde5f48a9f0
+...
+
+x = db.fetch_db("ISO 5749")
+=> nil
+
+# Fetching asynchronously
+
+# prepare queue for results
+results = Queue.new
+
+# references ot fetch
+refs = ["ISO 19011", "ISO 19115"]
+
+# fetch documents
+refs.each do |ref|
+  db.fetch_async(ref) do |doc|
+    results << [ref, doc]
+  end
+end
+
+# wait until documets fetching
+refs.size.times do
+  ref, doc = results.pop
+  # do thatever you need with result x
+end
 ----
 
 ==== Fetch by URN
@@ -192,6 +264,82 @@ bib.relation[1].bibitem.docidentifier[0].id
 [source,ruby]
 ----
 bib = db.fetch "ISO 19115-1, Amd 1"
+=> ["Chinese Standard", "GB/T 1.1"]
+[relaton-iso] ("ISO 19115-1") fetching...
+[relaton-iso] ("ISO 19115-1") found ISO 19115-1:2014
+[relaton-iso] ("ISO 19115-1/Amd 1") fetching...
+[relaton-iso] ("ISO 19115-1/Amd 1") found ISO 19115-1:2014/Amd 1:2018
+=> #<RelatonIsoBib::IsoBibliographicItem:0x007fb09b36d1b8
+...
+----
+
+==== Fetch all documents from cache
+
+`Relaton::Db#fetch_all(text = nil, edition: nil, year: nil)` - fetches all document from local cache
+
+* `text` - (String) filter entries by a text (optional)
+* `edition` - (String) filter entries by an edition (optional)
+* `year` - (Integer) filter entries by a year (optional)
+
+[source,ruby]
+----
+# query for all entries in a cahche
+
+items = db.fetch_all
+=> [#<RelatonIec::IecBibliographicItem:0x007facda8fdc28
+...
+
+items.size
+=> 6
+
+# query for all entries in a cahche for a certain string
+
+items = db.fetch_all("mathematical terminology")
+=> [#<RelatonIec::IecBibliographicItem:0x007ffeae5bd240
+...
+
+items.size
+=> 1
+
+items[0].docidentifier[0].id
+=> "IEC 60050-102:2007"
+
+# query for all entries in a cahche for a certain string and edition
+
+items = db.fetch_all("system", edition: "2")
+=> [#<RelatonIsoBib::IsoBibliographicItem:0x007ffebe2d1be8
+...
+
+items.size
+=> 1
+
+items[0].docidentifier[0].id
+=> "ISO 19011:2011"
+
+# query for all entries in a cahche for a certain string and year
+
+items = db.fetch_all("system", year: 2018)
+=> [#<RelatonIsoBib::IsoBibliographicItem:0x007ffeae645fa0
+...
+
+items.size
+=> 1
+
+items[0].docidentifier[0].id
+=> "ISO 19011 (all parts)"
+----
+
+=== Static DB
+
+This gem has a static DB which is distributed with it. Now the static contains documents:
+----
+ISO/IEC DIR 1 IEC SUP
+ISO/IEC DIR 1 ISO SUP
+ISO/IEC DIR 1
+ISO/IEC DIR 2 IEC
+ISO/IEC DIR 2 ISO
+ISO/IEC DIR IEC SUP
+ISO/IEC DIR 1 JTC SUP
 ----
 
 === Get document type
@@ -229,7 +377,7 @@ x.to_xml bibdata: true
 db.load_entry("ISO(ISO 19011)")
 => "<bibdata type="standard">
       ...
-    <?bibdata>"
+    </bibdata>"
 ----
 
 === Entry manipulation

data/lib/relaton.rb

@@ -1,8 +1,9 @@
-require_relative "relaton/util"
-require_relative "relaton/config"
+require "relaton/util"
+require "relaton/config"
 
-require_relative "relaton/db"
-require_relative "relaton/db_cache"
-require_relative "relaton/version"
-require_relative "relaton/registry"
-require_relative "relaton/processor"
+require "relaton/workers_pool"
+require "relaton/db"
+require "relaton/db_cache"
+require "relaton/version"
+require "relaton/registry"
+require "relaton/processor"

data/lib/relaton/config.rb

@@ -12,10 +12,13 @@ module Relaton
   end
 
   class Configuration
-    attr_accessor :logs
+    attr_accessor :logs, :use_api
 
     def initialize
-      @logs = %i(warning error)
+      @logs = %i(info error)
+
+      # @TODO change to true when we start using api.relaton.org
+      @use_api = false
     end
   end
 

data/lib/relaton/db.rb

@@ -10,83 +10,127 @@ module Relaton
     # @param local_cache [String] directory of local DB
     def initialize(global_cache, local_cache)
       @registry = Relaton::Registry.instance
-      @db = open_cache_biblio(global_cache, type: :global)
-      @local_db = open_cache_biblio(local_cache, type: :local)
-      @db_name = global_cache
-      @local_db_name = local_cache
-      static_db_name = File.expand_path "../relaton/static_cache", __dir__
-      @static_db = open_cache_biblio static_db_name
+      gpath = global_cache && File.expand_path(global_cache)
+      @db = open_cache_biblio(gpath, type: :global)
+      lpath = local_cache && File.expand_path(local_cache)
+      @local_db = open_cache_biblio(lpath, type: :local)
+      @static_db = open_cache_biblio File.expand_path("../relaton/static_cache",
+                                                      __dir__)
+      @queues = {}
     end
 
+    # Move global or local caches to anothe dirs
+    # @param new_dir [String, nil]
+    # @param type: [Symbol]
+    # @return [String, nil]
+    def mv(new_dir, type: :global)
+      case type
+      when :global
+        @db&.mv new_dir
+      when :local
+        @local_db&.mv new_dir
+      end
+    end
+
+    # Clear global and local databases
+    def clear
+      @db&.clear
+      @local_db&.clear
+    end
+
+    ##
     # The class of reference requested is determined by the prefix of the code:
-    # GB Standard for gbbib, IETF for ietfbib, ISO for isobib, IEC or IEV for iecbib,
+    # GB Standard for gbbib, IETF for ietfbib, ISO for isobib, IEC or IEV for
+    #   iecbib,
+    #
     # @param code [String] the ISO standard Code to look up (e.g. "ISO 9000")
     # @param year [String] the year the standard was published (optional)
-    # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
-    # @return [nil, RelatonBib::BibliographicItem, RelatonIsoBib::IsoBibliographicItem,
-    #   RelatonItu::ItuBibliographicItem, RelatonIetf::IetfBibliographicItem,
-    #   RelatonIec::IecBibliographicItem, RelatonIeee::IeeeBibliographicItem,
-    #   RelatonNist::NistBibliongraphicItem, RelatonGb::GbbibliographicItem,
-    #   RelatonOgc::OgcBibliographicItem, RelatonCalconnect::CcBibliographicItem]
+    #
+    # @param opts [Hash] options
+    # @option opts [Boolean] :all_parts If all-parts reference is required
+    # @option opts [Boolean] :keep_year If undated reference should return
+    #   actual reference with year
+    # @option opts [Integer] :retries (1) Number of network retries
+    #
+    # @return [nil, RelatonBib::BibliographicItem,
+    #   RelatonIsoBib::IsoBibliographicItem, RelatonItu::ItuBibliographicItem,
+    #   RelatonIetf::IetfBibliographicItem, RelatonIec::IecBibliographicItem,
+    #   RelatonIeee::IeeeBibliographicItem, RelatonNist::NistBibliongraphicItem,
+    #   RelatonGb::GbbibliographicItem, RelatonOgc::OgcBibliographicItem,
+    #   RelatonCalconnect::CcBibliographicItem, RelatinUn::UnBibliographicItem,
     #   RelatonBipm::BipmBibliographicItem, RelatonIho::IhoBibliographicItem,
-    #   RelatonOmg::OmgBibliographicItem RelatinUn::UnBibliographicItem,
-    #   RelatonW3c::W3cBibliographicItem
+    #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
+    ##
     def fetch(code, year = nil, opts = {})
       stdclass = standard_class(code) || return
       processor = @registry.processors[stdclass]
-      ref = processor.respond_to?(:urn_to_code) ? processor.urn_to_code(code)&.first : code
+      ref = if processor.respond_to?(:urn_to_code)
+              processor.urn_to_code(code)&.first
+            else code
+            end
       ref ||= code
-      cd = combine_doc ref, year, opts, stdclass
-      return cd if cd
+      result = combine_doc ref, year, opts, stdclass
+      result ||= check_bibliocache(ref, year, opts, stdclass)
+      result
+    end
 
-      check_bibliocache(ref, year, opts, stdclass)
+    # @see Relaton::Db#fetch
+    def fetch_db(code, year = nil, opts = {})
+      opts[:fetch_db] = true
+      fetch code, year, opts
     end
 
-    # @param code [String]
-    # @param year [String, nil]
-    # @param stdslass [String]
-    # @return [nil, RelatonBib::BibliographicItem, RelatonIsoBib::IsoBibliographicItem,
-    #   RelatonItu::ItuBibliographicItem, RelatonIetf::IetfBibliographicItem,
-    #   RelatonIec::IecBibliographicItem, RelatonIeee::IeeeBibliographicItem,
-    #   RelatonNist::NistBibliongraphicItem, RelatonGb::GbbibliographicItem,
-    #   RelatonOgc::OgcBibliographicItem, RelatonCalconnect::CcBibliographicItem]
-    #   RelatonBipm::BipmBibliographicItem, RelatonIho::IhoBibliographicItem,
-    #   RelatonOmg::OmgBibliographicItem RelatinUn::UnBibliographicItem,
-    #   RelatonW3c::W3cBibliographicItem
-    def combine_doc(code, year, opts, stdclass) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      if (refs = code.split " + ").size > 1
-        reltype = "derivedFrom"
-        reldesc = nil
-      elsif (refs = code.split ", ").size > 1
-        reltype = "complements"
-        reldesc = RelatonBib::FormattedString.new content: "amendment"
-      else return
+    # fetch all standards from DB
+    # @param test [String, nil]
+    # @param edition [String], nil
+    # @param year [Integer, nil]
+    # @return [Array]
+    def fetch_all(text = nil, edition: nil, year: nil)
+      result = @static_db.all do |file, yml|
+        search_yml file, yml, text, edition, year
+      end.compact
+      db = @db || @local_db
+      if db
+        result += db.all do |file, xml|
+          search_xml file, xml, text, edition, year
+        end.compact
       end
+      result
+    end
 
-      doc = @registry.processors[stdclass].hash_to_bib docid: { id: code }
-      ref = refs[0]
-      updates = check_bibliocache(ref, year, opts, stdclass)
-      doc.relation << RelatonBib::DocumentRelation.new(bibitem: updates, type: "updates") if updates
-      refs[1..-1].each_with_object(doc) do |c, d|
-        bib = check_bibliocache("#{ref}/#{c}", year, opts, stdclass)
-        if bib
-          d.relation << RelatonBib::DocumentRelation.new(type: reltype, description: reldesc, bibitem: bib)
+    # Fetch asynchronously
+    def fetch_async(code, year = nil, opts = {}, &_block) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      stdclass = standard_class code
+      if stdclass
+        unless @queues[stdclass]
+          processor = @registry.processors[stdclass]
+          wp = WorkersPool.new(processor.threads) { |args| yield fetch *args }
+          @queues[stdclass] = { queue: Queue.new, workers_pool: wp }
+          Thread.new { process_queue @queues[stdclass] }
         end
+        @queues[stdclass][:queue] << [code, year, opts]
+      else yield nil
       end
     end
 
     # @param code [String]
     # @param year [String, NilClass]
     # @param stdclass [Symbol, NilClass]
+    #
     # @param opts [Hash]
-    # @return [nil, RelatonBib::BibliographicItem, RelatonIsoBib::IsoBibliographicItem,
-    #   RelatonItu::ItuBibliographicItem, RelatonIetf::IetfBibliographicItem,
-    #   RelatonIec::IecBibliographicItem, RelatonIeee::IeeeBibliographicItem,
-    #   RelatonNist::NistBibliongraphicItem, RelatonGb::GbbibliographicItem,
-    #   RelatonOgc::OgcBibliographicItem, RelatonCalconnect::CcBibliographicItem]
+    # @option opts [Boolean] :all_parts If all-parts reference is required
+    # @option opts [Boolean] :keep_year If undated reference should return
+    #   actual reference with year
+    # @option opts [Integer] :retries (1) Number of network retries
+    #
+    # @return [nil, RelatonBib::BibliographicItem,
+    #   RelatonIsoBib::IsoBibliographicItem, RelatonItu::ItuBibliographicItem,
+    #   RelatonIetf::IetfBibliographicItem, RelatonIec::IecBibliographicItem,
+    #   RelatonIeee::IeeeBibliographicItem, RelatonNist::NistBibliongraphicItem,
+    #   RelatonGb::GbbibliographicItem, RelatonOgc::OgcBibliographicItem,
+    #   RelatonCalconnect::CcBibliographicItem, RelatinUn::UnBibliographicItem,
     #   RelatonBipm::BipmBibliographicItem, RelatonIho::IhoBibliographicItem,
-    #   RelatonOmg::OmgBibliographicItem RelatinUn::UnBibliographicItem,
-    #   RelatonW3c::W3cBibliographicItem
+    #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
     def fetch_std(code, year = nil, stdclass = nil, opts = {})
       std = nil
       @registry.processors.each do |name, processor|
@@ -137,6 +181,98 @@ module Relaton
 
     private
 
+    # @param file [String] file path
+    # @param yml [String] content in YAML format
+    # @param text [String, nil] text to serach
+    # @param edition [String, nil] edition to filter
+    # @param year [Integer, nil] year to filter
+    # @return [BibliographicItem, nil]
+    def search_yml(file, yml, text, edition, year)
+      item = search_edition_year(file, yml, edition, year)
+      return unless item
+
+      item if match_xml_text(item.to_xml(bibdata: true), text)
+    end
+
+    # @param file [String] file path
+    # @param xml [String] content in XML format
+    # @param text [String, nil] text to serach
+    # @param edition [String, nil] edition to filter
+    # @param year [Integer, nil] year to filter
+    # @return [BibliographicItem, nil]
+    def search_xml(file, xml, text, edition, year)
+      return unless text.nil? || match_xml_text(xml, text)
+
+      search_edition_year(file, xml, edition, year)
+    end
+
+    # @param file [String] file path
+    # @param content [String] content in XML or YAmL format
+    # @param edition [String, nil] edition to filter
+    # @param year [Integer, nil] year to filter
+    # @return [BibliographicItem, nil]
+    def search_edition_year(file, content, edition, year) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      processor = @registry.processors[standard_class(file.split("/")[-2])]
+      item = if file.match?(/xml$/) then processor.from_xml(content)
+             else processor.hash_to_bib(YAML.safe_load(content))
+             end
+      item if (edition.nil? || item.edition == edition) && (year.nil? ||
+        item.date.detect { |d| d.type == "published" && d.on(:year).to_s == year.to_s })
+    end
+
+    # @param xml [String] content in XML format
+    # @param text [String, nil] text to serach
+    # @return [Boolean]
+    def match_xml_text(xml, text)
+      %r{((?<attr>=((?<apstr>')|"))|>).*?#{text}.*?(?(<attr>)(?(<apstr>)'|")|<)}mi.match?(xml)
+    end
+
+    # @param code [String]
+    # @param year [String, nil]
+    # @param stdslass [String]
+    #
+    # @param opts [Hash] options
+    # @option opts [Boolean] :all_parts If all-parts reference is required
+    # @option opts [Boolean] :keep_year If undated reference should return
+    #   actual reference with year
+    # @option opts [Integer] :retries (1) Number of network retries
+    #
+    # @return [nil, RelatonBib::BibliographicItem,
+    #   RelatonIsoBib::IsoBibliographicItem, RelatonItu::ItuBibliographicItem,
+    #   RelatonIetf::IetfBibliographicItem, RelatonIec::IecBibliographicItem,
+    #   RelatonIeee::IeeeBibliographicItem, RelatonNist::NistBibliongraphicItem,
+    #   RelatonGb::GbbibliographicItem, RelatonOgc::OgcBibliographicItem,
+    #   RelatonCalconnect::CcBibliographicItem, RelatinUn::UnBibliographicItem,
+    #   RelatonBipm::BipmBibliographicItem, RelatonIho::IhoBibliographicItem,
+    #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
+    def combine_doc(code, year, opts, stdclass) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      if (refs = code.split " + ").size > 1
+        reltype = "derivedFrom"
+        reldesc = nil
+      elsif (refs = code.split ", ").size > 1
+        reltype = "complements"
+        reldesc = RelatonBib::FormattedString.new content: "amendment"
+      else return
+      end
+
+      doc = @registry.processors[stdclass].hash_to_bib docid: { id: code }
+      ref = refs[0]
+      updates = check_bibliocache(ref, year, opts, stdclass)
+      if updates
+        doc.relation << RelatonBib::DocumentRelation.new(bibitem: updates,
+                                                         type: "updates")
+      end
+      divider = stdclass == :relaton_itu ? " " : "/"
+      refs[1..-1].each_with_object(doc) do |c, d|
+        bib = check_bibliocache(ref + divider + c, year, opts, stdclass)
+        if bib
+          d.relation << RelatonBib::DocumentRelation.new(
+            type: reltype, description: reldesc, bibitem: bib
+          )
+        end
+      end
+    end
+
     # @param code [String] code of standard
     # @return [Symbol] standard class name
     def standard_class(code)
@@ -147,8 +283,8 @@ module Relaton
       allowed = @registry.processors.reduce([]) do |m, (_k, v)|
         m << v.prefix
       end
-      warn <<~WARN
-        #{code} does not have a recognised prefix: #{allowed.join(', ')}.
+      Util.log <<~WARN, :info
+        [relaton] #{code} does not have a recognised prefix: #{allowed.join(', ')}.
         See https://github.com/relaton/relaton/ for instructions on prefixing and wrapping document identifiers to disambiguate them.
       WARN
     end
@@ -157,7 +293,13 @@ module Relaton
     # Fofmat ID
     # @param code [String]
     # @param year [String]
+    #
     # @param opts [Hash]
+    # @option opts [Boolean] :all_parts If all-parts reference is required
+    # @option opts [Boolean] :keep_year If undated reference should return
+    #   actual reference with year
+    # @option opts [Integer] :retries (1) Number of network retries
+    #
     # @param stdClass [Symbol]
     # @return [Array<String>] docid and code
     def std_id(code, year, opts, stdclass)
@@ -180,79 +322,116 @@ module Relaton
 
     # @param entry [String] XML string
     # @param stdclass [Symbol]
-    # @param id [String] docid
-    # @return [nil, RelatonBib::BibliographicItem, RelatonIsoBib::IsoBibliographicItem,
-    #   RelatonItu::ItuBibliographicItem, RelatonIetf::IetfBibliographicItem,
-    #   RelatonIec::IecBibliographicItem, RelatonIeee::IeeeBibliographicItem,
-    #   RelatonNist::NistBibliongraphicItem, RelatonGb::GbbibliographicItem,
-    #   RelatonOgc::OgcBibliographicItem, RelatonCalconnect::CcBibliographicItem]
+    # @return [nil, RelatonBib::BibliographicItem,
+    #   RelatonIsoBib::IsoBibliographicItem, RelatonItu::ItuBibliographicItem,
+    #   RelatonIetf::IetfBibliographicItem, RelatonIec::IecBibliographicItem,
+    #   RelatonIeee::IeeeBibliographicItem, RelatonNist::NistBibliongraphicItem,
+    #   RelatonGb::GbbibliographicItem, RelatonOgc::OgcBibliographicItem,
+    #   RelatonCalconnect::CcBibliographicItem, RelatinUn::UnBibliographicItem,
     #   RelatonBipm::BipmBibliographicItem, RelatonIho::IhoBibliographicItem,
-    #   RelatonOmg::OmgBibliographicItem RelatinUn::UnBibliographicItem,
-    #   RelatonW3c::W3cBibliographicItem
-    def bib_retval(entry, stdclass, _id)
-      entry.match?(/^not_found/) ? nil : @registry.processors[stdclass].from_xml(entry)
+    #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
+    def bib_retval(entry, stdclass)
+      if entry.nil? || entry.match?(/^not_found/) then nil
+      else @registry.processors[stdclass].from_xml(entry)
+      end
     end
 
     # @param code [String]
     # @param year [String]
+    #
     # @param opts [Hash]
+    # @option opts [Boolean] :all_parts If all-parts reference is required
+    # @option opts [Boolean] :keep_year If undated reference should return
+    #   actual reference with year
+    # @option opts [Integer] :retries (1) Number of network retries
+    #
     # @param stdclass [Symbol]
-    # @return [nil, RelatonBib::BibliographicItem, RelatonIsoBib::IsoBibliographicItem,
-    #   RelatonItu::ItuBibliographicItem, RelatonIetf::IetfBibliographicItem,
-    #   RelatonIec::IecBibliographicItem, RelatonIeee::IeeeBibliographicItem,
-    #   RelatonNist::NistBibliongraphicItem, RelatonGb::GbbibliographicItem,
-    #   RelatonOgc::OgcBibliographicItem, RelatonCalconnect::CcBibliographicItem]
+    # @return [nil, RelatonBib::BibliographicItem,
+    #   RelatonIsoBib::IsoBibliographicItem, RelatonItu::ItuBibliographicItem,
+    #   RelatonIetf::IetfBibliographicItem, RelatonIec::IecBibliographicItem,
+    #   RelatonIeee::IeeeBibliographicItem, RelatonNist::NistBibliongraphicItem,
+    #   RelatonGb::GbbibliographicItem, RelatonOgc::OgcBibliographicItem,
+    #   RelatonCalconnect::CcBibliographicItem, RelatinUn::UnBibliographicItem,
     #   RelatonBipm::BipmBibliographicItem, RelatonIho::IhoBibliographicItem,
-    #   RelatonOmg::OmgBibliographicItem RelatinUn::UnBibliographicItem,
-    #   RelatonW3c::W3cBibliographicItem
+    #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
     def check_bibliocache(code, year, opts, stdclass) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
       id, searchcode = std_id(code, year, opts, stdclass)
       yaml = @static_db[id]
-      return @registry.processors[stdclass].hash_to_bib YAML.safe_load(yaml) if yaml
+      if yaml
+        return @registry.processors[stdclass].hash_to_bib YAML.safe_load(yaml)
+      end
 
       db = @local_db || @db
       altdb = @local_db && @db ? @db : nil
       if db.nil?
-        bibentry = new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
-        return bib_retval(bibentry, stdclass, id)
+        return if opts[:fetch_db]
+
+        bibentry = new_bib_entry(searchcode, year, opts, stdclass, db: db,
+                                                                   id: id)
+        return bib_retval(bibentry, stdclass)
       end
 
       db.delete(id) unless db.valid_entry?(id, year)
       if altdb
-        # db[id] ||= altdb[id]
-        db.clone_entry id, altdb
-        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
+        return bib_retval(altdb[id], stdclass) if opts[:fetch_db]
+
+        db.clone_entry id, altdb if altdb.valid_entry? id, year
+        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db,
+                                                                   id: id)
         altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
       else
-        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
+        return bib_retval(db[id], stdclass) if opts[:fetch_db]
+
+        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db,
+                                                                   id: id)
       end
-      bib_retval(db[id], stdclass, id)
+      bib_retval(db[id], stdclass)
     end
 
     # @param code [String]
     # @param year [String]
+    #
     # @param opts [Hash]
+    # @option opts [Boolean] :all_parts If all-parts reference is required
+    # @option opts [Boolean] :keep_year If undated reference should return
+    #   actual reference with year
+    # @option opts [Integer] :retries (1) Number of network retries
+    #
     # @param stdclass [Symbol]
     # @param db [Relaton::DbCache,`NilClass]
     # @param id [String] docid
     # @return [String]
     def new_bib_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-      bib = @registry.processors[stdclass].get(code, year, opts)
+      bib = net_retry(code, year, opts, stdclass, opts.fetch(:retries, 1))
       bib_id = bib&.docidentifier&.first&.id
 
       # when docid doesn't match bib's id then return a reference to bib's id
-      if args[:db] && args[:id] && bib_id && args[:id] !~ %r{#{Regexp.quote("(#{bib_id})")}}
+      if args[:db] && args[:id] &&
+          bib_id && args[:id] !~ %r{#{Regexp.quote("(#{bib_id})")}}
         bid = std_id(bib.docidentifier.first.id, nil, {}, stdclass).first
         args[:db][bid] ||= bib_entry bib
         "redirection #{bid}"
-      else
-        bib_entry bib
+      else bib_entry bib
       end
     end
 
-    # @param bib [RelatonGb::GbBibliongraphicItem, RelatonIsoBib::IsoBibliographicItem,
-    #   RelatonIetf::IetfBibliographicItem, RelatonItu::ItuBibliographicItem,
-    #   RelatonNist::NistBibliongraphicItem, RelatonOgc::OgcBibliographicItem]
+    # @raise [RelatonBib::RequestError]
+    def net_retry(code, year, opts, stdclass, retries)
+      @registry.processors[stdclass].get(code, year, opts)
+    rescue RelatonBib::RequestError => e
+      raise e unless retries > 1
+
+      net_retry(code, year, opts, stdclass, retries - 1)
+    end
+
+    # @param bib [RelatonBib::BibliographicItem,
+    #   RelatonIsoBib::IsoBibliographicItem, RelatonItu::ItuBibliographicItem,
+    #   RelatonIetf::IetfBibliographicItem, RelatonIec::IecBibliographicItem,
+    #   RelatonIeee::IeeeBibliographicItem, RelatonNist::NistBibliongraphicItem,
+    #   RelatonGb::GbbibliographicItem, RelatonOgc::OgcBibliographicItem,
+    #   RelatonCalconnect::CcBibliographicItem, RelatinUn::UnBibliographicItem,
+    #   RelatonBipm::BipmBibliographicItem, RelatonIho::IhoBibliographicItem,
+    #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
     # @return [String] XML or "not_found mm-dd-yyyy"
     def bib_entry(bib)
       if bib.respond_to? :to_xml
@@ -262,21 +441,33 @@ module Relaton
       end
     end
 
-    # @param dir [String] DB directory
+    # @param dir [String, nil] DB directory
     # @param type [Symbol]
     # @return [Relaton::DbCache, NilClass]
-    def open_cache_biblio(dir, type: :static)
+    def open_cache_biblio(dir, type: :static) # rubocop:disable Metrics/MethodLength
       return nil if dir.nil?
 
       db = DbCache.new dir, type == :static ? "yml" : "xml"
+      return db if type == :static
 
       Dir["#{dir}/*/"].each do |fdir|
-        next if type == :static || db.check_version?(fdir)
+        next if db.check_version?(fdir)
 
-        FileUtils.rm_rf(Dir.glob(fdir + "/*"), secure: true)
-        warn "[relaton] cache #{fdir}: version is obsolete and cache is cleared."
+        FileUtils.rm_rf(fdir, secure: true)
+        Util.log(
+          "[relaton] WARNING: cache #{fdir}: version is obsolete and cache is "\
+            "cleared.",
+          :warning
+        )
       end
       db
     end
+
+    # @param qwp [Hash]
+    # @option qwp [Queue] :queue The queue of references to fetch
+    # @option qwp [Relaton::WorkersPool] :workers_pool The pool of workers
+    def process_queue(qwp)
+      while args = qwp[:queue].pop; qwp[:workers_pool] << args end
+    end
   end
 end