relaton 1.7.9 → 1.8.pre2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b14a266fd6fd230fc37d8e2e6843bd48c37019136dc6006a27a89668ecf7af2
-  data.tar.gz: 3eb6ba9aaa798ab5b763328712e4051184ff2c9de1659791198ca8a7a21726c7
+  metadata.gz: 7613bdf8be768f76e3e24d8727210075cd8ad8d4e24924ec5d19d28473cd933f
+  data.tar.gz: 1332d1463afd07385d3aacb917a2f621b572e802b25ec64d5d5f87781b6c6ef6
 SHA512:
-  metadata.gz: 13a0f6f87426975dbf87e9eefd69d3f519e3a178452039eb9d036725e4c7bdd676ebe0d228a07b3512d296b6d4153cc1e88c72589baa702fb8ac1de717204783
-  data.tar.gz: e748db6725c7d027b38e1140dc60d74cdea86304f2380d9251cc9d07f143d0536e4b7e1bb63dea35ddd464e2969e08c03f6cde10a426f19e0a8959ecf7a6281f
+  metadata.gz: dd636d5da683476ff6abf8a0c686bd9b7112cbbbda7c19f96249b75a6ecb8a4cb4c5764eeead68e949003b8f460574b50fabbcbe2f838ff9f104cb52adda45ba
+  data.tar.gz: 75ec0ed77c187b48baef584702b10cd5add0938325a19a40eac6e0217c86b7f769f30ad2561c612ab4fd2c0905c03e8c94797c92ce28a034215cf88bfc8d5f09

data/.github/workflows/rake.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
+        ruby: [ '2.7', '2.6', '2.5' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
         include:

data/.rubocop.yml

@@ -5,6 +5,6 @@
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5
 Rails:
   Enabled: false

data/lib/relaton.rb

@@ -1,5 +1,6 @@
 require "relaton/util"
 require "relaton/config"
+require "yaml"
 
 require "relaton/workers_pool"
 require "relaton/db"
@@ -7,3 +8,5 @@ require "relaton/db_cache"
 require "relaton/version"
 require "relaton/registry"
 require "relaton/processor"
+require "relaton/registry"
+require "relaton/db_cache"

data/lib/relaton/config.rb

@@ -12,13 +12,15 @@ module Relaton
   end
 
   class Configuration
-    attr_accessor :logs, :use_api
+    attr_accessor :logs, :use_api, :api_host, :api_mode
 
     def initialize
       @logs = %i(info error) # allowed values: :info, :warning, :error, :debug
 
       # @TODO change to true when we start using api.relaton.org
-      @use_api = false
+      @use_api = true
+      @api_mode = false
+      @api_host = nil # "http://0.0.0.0:9292"
     end
   end
 

data/lib/relaton/db.rb

@@ -1,7 +1,3 @@
-require "yaml"
-require_relative "registry"
-require_relative "db_cache"
-
 module Relaton
   class RelatonError < StandardError; end
 
@@ -10,12 +6,16 @@ module Relaton
     # @param local_cache [String] directory of local DB
     def initialize(global_cache, local_cache)
       @registry = Relaton::Registry.instance
-      gpath = global_cache && File.expand_path(global_cache)
+      if Relaton.configuration.api_mode
+        gpath = global_cache
+        lpath = local_cache
+      else
+        gpath = global_cache && File.expand_path(global_cache)
+        lpath = local_cache && File.expand_path(local_cache)
+      end
       @db = open_cache_biblio(gpath, type: :global)
-      lpath = local_cache && File.expand_path(local_cache)
       @local_db = open_cache_biblio(lpath, type: :local)
-      @static_db = open_cache_biblio File.expand_path("../relaton/static_cache",
-                                                      __dir__)
+      @static_db = open_cache_biblio File.expand_path "../relaton/static_cache", __dir__
       @queues = {}
     end
 
@@ -25,10 +25,8 @@ module Relaton
     # @return [String, nil]
     def mv(new_dir, type: :global)
       case type
-      when :global
-        @db&.mv new_dir
-      when :local
-        @local_db&.mv new_dir
+      when :global then @db&.mv new_dir
+      when :local then @local_db&.mv new_dir
       end
     end
 
@@ -89,22 +87,18 @@ module Relaton
       result = @static_db.all do |file, yml|
         search_yml file, yml, text, edition, year
       end.compact
-      db = @db || @local_db
-      if db
-        result += db.all do |file, xml|
-          search_xml file, xml, text, edition, year
-        end.compact
+      if (db = @db || @local_db)
+        result += db.all { |f, x| search_xml f, x, text, edition, year }.compact
       end
       result
     end
 
     # Fetch asynchronously
     def fetch_async(code, year = nil, opts = {}, &_block) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      stdclass = standard_class code
-      if stdclass
+      if (stdclass = standard_class code)
         unless @queues[stdclass]
           processor = @registry.processors[stdclass]
-          wp = WorkersPool.new(processor.threads) { |args| yield fetch *args }
+          wp = WorkersPool.new(processor.threads) { |args| yield fetch(*args) }
           @queues[stdclass] = { queue: Queue.new, workers_pool: wp }
           Thread.new { process_queue @queues[stdclass] }
         end
@@ -173,14 +167,31 @@ module Relaton
     def to_xml
       db = @local_db || @db || return
       Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
-        xml.documents do
-          xml.parent.add_child db.all.join(" ")
-        end
+        xml.documents { xml.parent.add_child db.all.join(" ") }
       end.to_xml
     end
 
     private
 
+    #
+    # @param code [String]
+    # @param year [String]
+    #
+    # @param opts [Hash]
+    # @option opts [Boolean] :all_parts If all-parts reference is required
+    # @option opts [Boolean] :keep_year If undated reference should return
+    #   actual reference with year
+    #
+    # @param stdclass [Symbol]
+    def fetch_api(code, year, opts, stdclass)
+      return unless Relaton.configuration.use_api
+
+      params = opts.merge(code: code, year: year).map { |k, v| "#{k}=#{v}" }.join "&"
+      url = "#{Relaton.configuration.api_host}/api/v1/fetch?#{params}"
+      rsp = Net::HTTP.get_response URI(url)
+      @registry.processors[stdclass].from_xml rsp.body if rsp.code == "200"
+    end
+
     # @param file [String] file path
     # @param yml [String] content in YAML format
     # @param text [String, nil] text to serach
@@ -207,7 +218,7 @@ module Relaton
     end
 
     # @param file [String] file path
-    # @param content [String] content in XML or YAmL format
+    # @param content [String] content in XML or YAML format
     # @param edition [String, nil] edition to filter
     # @param year [Integer, nil] year to filter
     # @return [BibliographicItem, nil]
@@ -220,9 +231,14 @@ module Relaton
         item.date.detect { |d| d.type == "published" && d.on(:year).to_s == year.to_s })
     end
 
+    #
+    # Look up text in the XML elements attributes and content
+    #
     # @param xml [String] content in XML format
     # @param text [String, nil] text to serach
+    #
     # @return [Boolean]
+    #
     def match_xml_text(xml, text)
       %r{((?<attr>=((?<apstr>')|"))|>).*?#{text}.*?(?(<attr>)(?(<apstr>)'|")|<)}mi.match?(xml)
     end
@@ -256,19 +272,15 @@ module Relaton
       end
 
       doc = @registry.processors[stdclass].hash_to_bib docid: { id: code }
-      ref = refs[0]
-      updates = check_bibliocache(ref, year, opts, stdclass)
+      updates = check_bibliocache(refs[0], year, opts, stdclass)
       if updates
-        doc.relation << RelatonBib::DocumentRelation.new(bibitem: updates,
-                                                         type: "updates")
+        doc.relation << RelatonBib::DocumentRelation.new(bibitem: updates, type: "updates")
       end
       divider = stdclass == :relaton_itu ? " " : "/"
       refs[1..-1].each_with_object(doc) do |c, d|
-        bib = check_bibliocache(ref + divider + c, year, opts, stdclass)
+        bib = check_bibliocache(refs[0] + divider + c, year, opts, stdclass)
         if bib
-          d.relation << RelatonBib::DocumentRelation.new(
-            type: reltype, description: reldesc, bibitem: bib
-          )
+          d.relation << RelatonBib::DocumentRelation.new(type: reltype, description: reldesc, bibitem: bib)
         end
       end
     end
@@ -280,9 +292,7 @@ module Relaton
         return name if /^(urn:)?#{processor.prefix}/i.match?(code) ||
           processor.defaultprefix.match(code)
       end
-      allowed = @registry.processors.reduce([]) do |m, (_k, v)|
-        m << v.prefix
-      end
+      allowed = @registry.processors.reduce([]) { |m, (_k, v)| m << v.prefix }
       Util.log <<~WARN, :info
         [relaton] #{code} does not have a recognised prefix: #{allowed.join(', ')}.
         See https://github.com/relaton/relaton/ for instructions on prefixing and wrapping document identifiers to disambiguate them.
@@ -331,8 +341,8 @@ module Relaton
     #   RelatonBipm::BipmBibliographicItem, RelatonIho::IhoBibliographicItem,
     #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
     def bib_retval(entry, stdclass)
-      if entry.nil? || entry.match?(/^not_found/) then nil
-      else @registry.processors[stdclass].from_xml(entry)
+      unless entry.nil? || entry.match?(/^not_found/)
+        @registry.processors[stdclass].from_xml(entry)
       end
     end
 
@@ -356,8 +366,7 @@ module Relaton
     #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
     def check_bibliocache(code, year, opts, stdclass) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
       id, searchcode = std_id(code, year, opts, stdclass)
-      yaml = @static_db[id]
-      if yaml
+      if (yaml = @static_db[id])
         return @registry.processors[stdclass].hash_to_bib YAML.safe_load(yaml)
       end
 
@@ -366,8 +375,7 @@ module Relaton
       if db.nil?
         return if opts[:fetch_db]
 
-        bibentry = new_bib_entry(searchcode, year, opts, stdclass, db: db,
-                                                                   id: id)
+        bibentry = new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
         return bib_retval(bibentry, stdclass)
       end
 
@@ -376,18 +384,17 @@ module Relaton
         return bib_retval(altdb[id], stdclass) if opts[:fetch_db]
 
         db.clone_entry id, altdb if altdb.valid_entry? id, year
-        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db,
-                                                                   id: id)
+        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
         altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
       else
         return bib_retval(db[id], stdclass) if opts[:fetch_db]
 
-        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db,
-                                                                   id: id)
+        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
       end
       bib_retval(db[id], stdclass)
     end
 
+    #
     # @param code [String]
     # @param year [String]
     #
@@ -400,7 +407,9 @@ module Relaton
     # @param stdclass [Symbol]
     # @param db [Relaton::DbCache,`NilClass]
     # @param id [String] docid
+    #
     # @return [String]
+    #
     def new_bib_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
       bib = net_retry(code, year, opts, stdclass, opts.fetch(:retries, 1))
       bib_id = bib&.docidentifier&.first&.id
@@ -415,8 +424,27 @@ module Relaton
       end
     end
 
+    #
+    # @param code [String]
+    # @param year [String]
+    #
+    # @param opts [Hash]
+    # @option opts [Boolean] :all_parts If all-parts reference is required
+    # @option opts [Boolean] :keep_year If undated reference should return
+    #   actual reference with year
+    #
+    # @param stdclass [Symbol]
+    # @param retries [Integer] remain Number of network retries
+    #
     # @raise [RelatonBib::RequestError]
+    # @return [RelatonBib::BibliographicItem]
+    #
     def net_retry(code, year, opts, stdclass, retries)
+      doc = fetch_api code, year, opts, stdclass
+      return doc if doc
+
+      @registry.processors[stdclass].get(code, year, opts)
+    rescue Errno::ECONNREFUSED
       @registry.processors[stdclass].get(code, year, opts)
     rescue RelatonBib::RequestError => e
       raise e unless retries > 1
@@ -434,11 +462,7 @@ module Relaton
     #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
     # @return [String] XML or "not_found mm-dd-yyyy"
     def bib_entry(bib)
-      if bib.respond_to? :to_xml
-        bib.to_xml(bibdata: true)
-      else
-        "not_found #{Date.today}"
-      end
+      bib.respond_to?(:to_xml) ? bib.to_xml(bibdata: true) : "not_found #{Date.today}"
     end
 
     # @param dir [String, nil] DB directory
@@ -456,8 +480,7 @@ module Relaton
         FileUtils.rm_rf(fdir, secure: true)
         Util.log(
           "[relaton] WARNING: cache #{fdir}: version is obsolete and cache is "\
-            "cleared.",
-          :warning
+            "cleared.", :warning
         )
       end
       db
@@ -469,5 +492,36 @@ module Relaton
     def process_queue(qwp)
       while args = qwp[:queue].pop; qwp[:workers_pool] << args end
     end
+
+    class << self
+      # Initialse and return relaton instance, with local and global cache names
+      # local_cache: local cache name; none created if nil; "relaton" created
+      # if empty global_cache: boolean to create global_cache
+      # flush_caches: flush caches
+      def init_bib_caches(**opts) # rubocop:disable Metrics/CyclomaticComplexity
+        globalname = global_bibliocache_name if opts[:global_cache]
+        localname = local_bibliocache_name(opts[:local_cache])
+        flush_caches globalname, localname if opts[:flush_caches]
+        Relaton::Db.new(globalname, localname)
+      end
+
+      private
+
+      def flush_caches(gcache, lcache)
+        FileUtils.rm_rf gcache unless gcache.nil?
+        FileUtils.rm_rf lcache unless lcache.nil?
+      end
+
+      def global_bibliocache_name
+        Relaton.configuration.api_mode ? "cache" : "#{Dir.home}/.relaton/cache"
+      end
+
+      def local_bibliocache_name(cachename)
+        return nil if Relaton.configuration.api_mode || cachename.nil?
+
+        cachename = "relaton" if cachename.empty?
+        "#{cachename}/cache"
+      end
+    end
   end
 end

data/lib/relaton/db_cache.rb

@@ -1,5 +1,6 @@
 require "fileutils"
 require "timeout"
+require "relaton/storage"
 
 module Relaton
   class DbCache
@@ -10,16 +11,15 @@ module Relaton
     def initialize(dir, ext = "xml")
       @dir = dir
       @ext = ext
-      FileUtils::mkdir_p @dir
-      # file_version = "#{@dir}/version"
-      # set_version # unless File.exist? file_version
+      @storage = Storage.instance
+      FileUtils::mkdir_p @dir unless Relaton.configuration.api_mode
     end
 
     # Move caches to anothe dir
     # @param new_dir [String, nil]
     # @return [String, nil]
     def mv(new_dir)
-      return unless new_dir && @ext == "xml"
+      return unless new_dir && @ext == "xml" && !Relaton.configuration.api_mode
 
       if File.exist? new_dir
         warn "[relaton] WARNING: target directory exists \"#{new_dir}\""
@@ -32,7 +32,9 @@ module Relaton
 
     # Clear database
     def clear
-      FileUtils.rm_rf Dir.glob "#{dir}/*" if @ext == "xml" # unless it's static DB
+      return if Relaton.configuration.api_mode
+
+      FileUtils.rm_rf Dir.glob "#{dir}/*" if @ext == "xml" # if it's static DB
     end
 
     # Save item
@@ -43,21 +45,10 @@ module Relaton
         delete key
         return
       end
-
-      prefix_dir = "#{@dir}/#{prefix(key)}"
-      FileUtils::mkdir_p prefix_dir unless Dir.exist? prefix_dir
-      set_version prefix_dir
-      file_safe_write "#{filename(key)}.#{ext(value)}", value
-    end
-
-    # @param value [String]
-    # @return [String]
-    def ext(value)
-      case value
-      when /^not_found/ then "notfound"
-      when /^redirection/ then "redirect"
-      else @ext
-      end
+      /^(?<pref>[^(]+)(?=\()/ =~ key.downcase
+      prefix_dir = "#{@dir}/#{pref}"
+      file = "#{filename(key)}.#{ext(value)}"
+      @storage.save prefix_dir, file, value
     end
 
     # Read item
@@ -86,7 +77,7 @@ module Relaton
       value = self[key]
       return unless value
 
-      if value.match? /^not_found/
+      if value.match?(/^not_found/)
         value.match(/\d{4}-\d{2}-\d{2}/).to_s
       else
         doc = Nokogiri::XML value
@@ -96,41 +87,21 @@ module Relaton
 
     # Returns all items
     # @return [Array<String>]
-    def all
-      Dir.glob("#{@dir}/**/*.{xml,yml,yaml}").sort.map do |f|
-        content = File.read(f, encoding: "utf-8")
-        block_given? ? yield(f, content) : content
-      end
+    def all(&block)
+      @storage.all(@dir, &block)
     end
 
     # Delete item
     # @param key [String]
     def delete(key)
-      file = filename key
-      f = search_ext(file)
-      File.delete f if f
+      @storage.delete filename(key)
     end
 
     # Check if version of the DB match to the gem grammar hash.
     # @param fdir [String] dir pathe to flover cache
-    # @return [TrueClass, FalseClass]
+    # @return [Boolean]
     def check_version?(fdir)
-      version_dir = fdir + "/version"
-      return false unless File.exist? version_dir
-
-      v = File.read version_dir, encoding: "utf-8"
-      v.strip == grammar_hash(fdir)
-    end
-
-    # Set version of the DB to the gem grammar hash.
-    # @param fdir [String] dir pathe to flover cache
-    # @return [Relaton::DbCache]
-    def set_version(fdir)
-      file_version = "#{fdir}/version"
-      unless File.exist? file_version
-        file_safe_write file_version, grammar_hash(fdir)
-      end
-      self
+      @storage.check_version? fdir
     end
 
     # if cached reference is undated, expire it after 60 days
@@ -144,112 +115,54 @@ module Relaton
       year || Date.today - date < 60
     end
 
-    protected
-
-    # @param fdir [String] dir pathe to flover cache
-    # @return [String]
-    def grammar_hash(fdir)
-      type = fdir.split("/").last
-      Relaton::Registry.instance.by_type(type)&.grammar_hash
-    end
-
     # Reads file by a key
     #
     # @param key [String]
     # @return [String, NilClass]
     def get(key)
-      file = filename key
-      return unless (f = search_ext(file))
-
-      File.read(f, encoding: "utf-8")
+      @storage.get filename(key), static: @ext == "yml"
     end
 
     private
 
-    # Check if a file content is redirection
-    #
-    # @prarm value [String] file content
-    # @return [String, NilClass] redirection code or nil
-    def redirect?(value)
-      %r{redirection\s(?<code>.*)} =~ value
-      code
+    # @param value [String]
+    # @return [String]
+    def ext(value)
+      case value
+      when /^not_found/ then "notfound"
+      when /^redirection/ then "redirect"
+      else @ext
+      end
     end
 
     # Return item's file name
     # @param key [String]
     # @return [String]
     def filename(key)
-      prefcode = key.downcase.match /^(?<prefix>[^\(]+)\((?<code>[^\)]+)/
+      prefcode = key.downcase.match(/^(?<prefix>[^(]+)\((?<code>[^)]+)/)
       fn = if prefcode
-             "#{prefcode[:prefix]}/#{prefcode[:code].gsub(/[-:\s\/\()]/, '_').squeeze('_')}"
+             "#{prefcode[:prefix]}/#{prefcode[:code].gsub(/[-:\s\/()]/, '_')
+               .squeeze('_')}"
            else
              key.gsub(/[-:\s]/, "_")
            end
       "#{@dir}/#{fn.sub(/(,|_$)/, '')}"
     end
 
+    # Check if a file content is redirection
     #
-    # Checks if there is file with xml or txt extension and return filename with
-    # the extension.
-    #
-    # @param file [String]
-    # @return [String, NilClass]
-    def search_ext(file)
-      if File.exist?("#{file}.#{@ext}")
-        "#{file}.#{@ext}"
-      elsif File.exist? "#{file}.notfound"
-        "#{file}.notfound"
-      elsif File.exist? "#{file}.redirect"
-        "#{file}.redirect"
-      end
+    # @prarm value [String] file content
+    # @return [String, NilClass] redirection code or nil
+    def redirect?(value)
+      %r{redirection\s(?<code>.*)} =~ value
+      code
     end
 
     # Return item's subdir
     # @param key [String]
     # @return [String]
-    def prefix(key)
-      key.downcase.match(/^[^\(]+(?=\()/).to_s
-    end
-
-    # @param file [String]
-    # @content [String]
-    def file_safe_write(file, content)
-      File.open file, File::RDWR | File::CREAT, encoding: "UTF-8" do |f|
-        Timeout.timeout(10) { f.flock File::LOCK_EX }
-        f.write content
-      end
-    end
-
-    class << self
-      private
-
-      def global_bibliocache_name
-        "#{Dir.home}/.relaton/cache"
-      end
-
-      def local_bibliocache_name(cachename)
-        return nil if cachename.nil?
-
-        cachename = "relaton" if cachename.empty?
-        "#{cachename}/cache"
-      end
-
-      public
-
-      # Initialse and return relaton instance, with local and global cache names
-      # local_cache: local cache name; none created if nil; "relaton" created
-      # if empty global_cache: boolean to create global_cache
-      # flush_caches: flush caches
-      def init_bib_caches(opts) # rubocop:disable Metrics/CyclomaticComplexity
-        globalname = global_bibliocache_name if opts[:global_cache]
-        localname = local_bibliocache_name(opts[:local_cache])
-        localname = "relaton" if localname&.empty?
-        if opts[:flush_caches]
-          FileUtils.rm_rf globalname unless globalname.nil?
-          FileUtils.rm_rf localname unless localname.nil?
-        end
-        Relaton::Db.new(globalname, localname)
-      end
-    end
+    # def prefix(key)
+    #   key.downcase.match(/^[^(]+(?=\()/).to_s
+    # end
   end
 end