relaton 1.8.pre3 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d954779afa85cdbe30b00b6780f4fe2ea261839da0f82855dca6e63fe8be77d
-  data.tar.gz: e7549980e5771e74412291428b8010ad4d6a7720526b98169b97dac82539e909
+  metadata.gz: 9a47a3437f5f8afb131ea6cf2a0cf1714875f45e3bd3881c17bc8c7ab9cc2101
+  data.tar.gz: b2a25232edd0d71d5597c06c011362771326bdf3369272e9ca5c862b76947427
 SHA512:
-  metadata.gz: 466c2e7c24c972df07bf9548b0dac26ec0eb2c03d7c8716abba50cc7a95f83dd58c903b3b99a99fbddcb3afa1d68518c93082acee68034ec3d3230e10d4f0b25
-  data.tar.gz: 3f4fb6fd8750375862ae109d8d50654372fa3d8e1b6b8eb966e827a026cd4c99f1790a68068c6785a93c6a2c459b6c1506a1768aef881025fa12914cffd0708b
+  metadata.gz: 2c99ab3fe479d11714a2848656076685057f9169d59d548530a60e8405d8be23d7f1a04b79744e6fbd112d4495e2cf5d1451d7e3f769fa0bb8844874281dee72
+  data.tar.gz: 48d30e142c8a611f97ebc9eb0aed6f3619ec2784e9856b1e659473b52c48fdae77c131c29241087c8a31e59623ed3830e383d75103fe42e18416a0a06c251c66

data/.github/workflows/rake.yml

@@ -16,19 +16,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '2.6', '2.5' ]
+        ruby: [ '3.0', '2.7', '2.6', '2.5' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
-        include:
-          - ruby: '3.0'
-            os: 'ubuntu-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'windows-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'macos-latest'
-            experimental: true
     steps:
       - uses: actions/checkout@v2
         with:

data/lib/relaton/config.rb

@@ -12,15 +12,13 @@ module Relaton
   end
 
   class Configuration
-    attr_accessor :logs, :use_api, :api_host, :api_mode
+    attr_accessor :logs, :use_api
 
     def initialize
       @logs = %i(info error) # allowed values: :info, :warning, :error, :debug
 
       # @TODO change to true when we start using api.relaton.org
-      @use_api = true
-      @api_mode = false
-      @api_host = "https://api.relaton.org/api/v1"
+      @use_api = false
     end
   end
 

data/lib/relaton/db.rb

@@ -1,21 +1,21 @@
+require "yaml"
+require_relative "registry"
+require_relative "db_cache"
+
 module Relaton
-  # class RelatonError < StandardError; end
+  class RelatonError < StandardError; end
 
   class Db
     # @param global_cache [String] directory of global DB
     # @param local_cache [String] directory of local DB
-    def initialize(global_cache, local_cache) # rubocop:disable Metrics/MethodLength
+    def initialize(global_cache, local_cache)
       @registry = Relaton::Registry.instance
-      if Relaton.configuration.api_mode
-        gpath = global_cache
-        lpath = local_cache
-      else
-        gpath = global_cache && File.expand_path(global_cache)
-        lpath = local_cache && File.expand_path(local_cache)
-      end
+      gpath = global_cache && File.expand_path(global_cache)
       @db = open_cache_biblio(gpath, type: :global)
+      lpath = local_cache && File.expand_path(local_cache)
       @local_db = open_cache_biblio(lpath, type: :local)
-      @static_db = open_cache_biblio File.expand_path "../relaton/static_cache", __dir__
+      @static_db = open_cache_biblio File.expand_path("../relaton/static_cache",
+                                                      __dir__)
       @queues = {}
     end
 
@@ -25,8 +25,10 @@ module Relaton
     # @return [String, nil]
     def mv(new_dir, type: :global)
       case type
-      when :global then @db&.mv new_dir
-      when :local then @local_db&.mv new_dir
+      when :global
+        @db&.mv new_dir
+      when :local
+        @local_db&.mv new_dir
       end
     end
 
@@ -87,15 +89,19 @@ module Relaton
       result = @static_db.all do |file, yml|
         search_yml file, yml, text, edition, year
       end.compact
-      if (db = @db || @local_db)
-        result += db.all { |f, x| search_xml f, x, text, edition, year }.compact
+      db = @db || @local_db
+      if db
+        result += db.all do |file, xml|
+          search_xml file, xml, text, edition, year
+        end.compact
       end
       result
     end
 
     # Fetch asynchronously
     def fetch_async(code, year = nil, opts = {}, &_block) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      if (stdclass = standard_class code)
+      stdclass = standard_class code
+      if stdclass
         unless @queues[stdclass]
           processor = @registry.processors[stdclass]
           wp = WorkersPool.new(processor.threads) { |args| yield fetch(*args) }
@@ -167,43 +173,14 @@ module Relaton
     def to_xml
       db = @local_db || @db || return
       Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
-        xml.documents { xml.parent.add_child db.all.join(" ") }
+        xml.documents do
+          xml.parent.add_child db.all.join(" ")
+        end
       end.to_xml
     end
 
     private
 
-    #
-    # @param code [String]
-    # @param year [String]
-    #
-    # @param opts [Hash]
-    # @option opts [Boolean] :all_parts If all-parts reference is required
-    # @option opts [Boolean] :keep_year If undated reference should return
-    #   actual reference with year
-    #
-    # @param stdclass [Symbol]
-    def fetch_api(code, year, opts, stdclass)
-      return unless Relaton.configuration.use_api
-
-      url = "#{Relaton.configuration.api_host}/document?#{params(code, year, opts)}"
-      rsp = Net::HTTP.get_response URI(url)
-      @registry.processors[stdclass].from_xml rsp.body if rsp.code == "200"
-    end
-
-    #
-    # Make string of parametrs
-    #
-    # @param [String] code
-    # @param [String] year
-    # @param [Hash] opts
-    #
-    # @return [String]
-    #
-    def params(code, year, opts)
-      opts.merge(code: code, year: year).map { |k, v| "#{k}=#{v}" }.join "&"
-    end
-
     # @param file [String] file path
     # @param yml [String] content in YAML format
     # @param text [String, nil] text to serach
@@ -230,7 +207,7 @@ module Relaton
     end
 
     # @param file [String] file path
-    # @param content [String] content in XML or YAML format
+    # @param content [String] content in XML or YAmL format
     # @param edition [String, nil] edition to filter
     # @param year [Integer, nil] year to filter
     # @return [BibliographicItem, nil]
@@ -243,14 +220,9 @@ module Relaton
         item.date.detect { |d| d.type == "published" && d.on(:year).to_s == year.to_s })
     end
 
-    #
-    # Look up text in the XML elements attributes and content
-    #
     # @param xml [String] content in XML format
     # @param text [String, nil] text to serach
-    #
     # @return [Boolean]
-    #
     def match_xml_text(xml, text)
       %r{((?<attr>=((?<apstr>')|"))|>).*?#{text}.*?(?(<attr>)(?(<apstr>)'|")|<)}mi.match?(xml)
     end
@@ -284,15 +256,19 @@ module Relaton
       end
 
       doc = @registry.processors[stdclass].hash_to_bib docid: { id: code }
-      updates = check_bibliocache(refs[0], year, opts, stdclass)
+      ref = refs[0]
+      updates = check_bibliocache(ref, year, opts, stdclass)
       if updates
-        doc.relation << RelatonBib::DocumentRelation.new(bibitem: updates, type: "updates")
+        doc.relation << RelatonBib::DocumentRelation.new(bibitem: updates,
+                                                         type: "updates")
       end
       divider = stdclass == :relaton_itu ? " " : "/"
       refs[1..-1].each_with_object(doc) do |c, d|
-        bib = check_bibliocache(refs[0] + divider + c, year, opts, stdclass)
+        bib = check_bibliocache(ref + divider + c, year, opts, stdclass)
         if bib
-          d.relation << RelatonBib::DocumentRelation.new(type: reltype, description: reldesc, bibitem: bib)
+          d.relation << RelatonBib::DocumentRelation.new(
+            type: reltype, description: reldesc, bibitem: bib
+          )
         end
       end
     end
@@ -304,7 +280,9 @@ module Relaton
         return name if /^(urn:)?#{processor.prefix}/i.match?(code) ||
           processor.defaultprefix.match(code)
       end
-      allowed = @registry.processors.reduce([]) { |m, (_k, v)| m << v.prefix }
+      allowed = @registry.processors.reduce([]) do |m, (_k, v)|
+        m << v.prefix
+      end
       Util.log <<~WARN, :info
         [relaton] #{code} does not have a recognised prefix: #{allowed.join(', ')}.
         See https://github.com/relaton/relaton/ for instructions on prefixing and wrapping document identifiers to disambiguate them.
@@ -353,8 +331,8 @@ module Relaton
     #   RelatonBipm::BipmBibliographicItem, RelatonIho::IhoBibliographicItem,
     #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
     def bib_retval(entry, stdclass)
-      unless entry.nil? || entry.match?(/^not_found/)
-        @registry.processors[stdclass].from_xml(entry)
+      if entry.nil? || entry.match?(/^not_found/) then nil
+      else @registry.processors[stdclass].from_xml(entry)
       end
     end
 
@@ -378,7 +356,8 @@ module Relaton
     #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
     def check_bibliocache(code, year, opts, stdclass) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
       id, searchcode = std_id(code, year, opts, stdclass)
-      if (yaml = @static_db[id])
+      yaml = @static_db[id]
+      if yaml
         return @registry.processors[stdclass].hash_to_bib YAML.safe_load(yaml)
       end
 
@@ -387,7 +366,8 @@ module Relaton
       if db.nil?
         return if opts[:fetch_db]
 
-        bibentry = new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
+        bibentry = new_bib_entry(searchcode, year, opts, stdclass, db: db,
+                                                                   id: id)
         return bib_retval(bibentry, stdclass)
       end
 
@@ -396,17 +376,18 @@ module Relaton
         return bib_retval(altdb[id], stdclass) if opts[:fetch_db]
 
         db.clone_entry id, altdb if altdb.valid_entry? id, year
-        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
+        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db,
+                                                                   id: id)
         altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
       else
         return bib_retval(db[id], stdclass) if opts[:fetch_db]
 
-        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
+        db[id] ||= new_bib_entry(searchcode, year, opts, stdclass, db: db,
+                                                                   id: id)
       end
       bib_retval(db[id], stdclass)
     end
 
-    #
     # @param code [String]
     # @param year [String]
     #
@@ -419,9 +400,7 @@ module Relaton
     # @param stdclass [Symbol]
     # @param db [Relaton::DbCache,`NilClass]
     # @param id [String] docid
-    #
     # @return [String]
-    #
     def new_bib_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
       bib = net_retry(code, year, opts, stdclass, opts.fetch(:retries, 1))
       bib_id = bib&.docidentifier&.first&.id
@@ -436,27 +415,8 @@ module Relaton
       end
     end
 
-    #
-    # @param code [String]
-    # @param year [String]
-    #
-    # @param opts [Hash]
-    # @option opts [Boolean] :all_parts If all-parts reference is required
-    # @option opts [Boolean] :keep_year If undated reference should return
-    #   actual reference with year
-    #
-    # @param stdclass [Symbol]
-    # @param retries [Integer] remain Number of network retries
-    #
     # @raise [RelatonBib::RequestError]
-    # @return [RelatonBib::BibliographicItem]
-    #
     def net_retry(code, year, opts, stdclass, retries)
-      doc = fetch_api code, year, opts, stdclass
-      return doc if doc
-
-      @registry.processors[stdclass].get(code, year, opts)
-    rescue Errno::ECONNREFUSED
       @registry.processors[stdclass].get(code, year, opts)
     rescue RelatonBib::RequestError => e
       raise e unless retries > 1
@@ -474,7 +434,11 @@ module Relaton
     #   RelatonOmg::OmgBibliographicItem, RelatonW3c::W3cBibliographicItem]
     # @return [String] XML or "not_found mm-dd-yyyy"
     def bib_entry(bib)
-      bib.respond_to?(:to_xml) ? bib.to_xml(bibdata: true) : "not_found #{Date.today}"
+      if bib.respond_to? :to_xml
+        bib.to_xml(bibdata: true)
+      else
+        "not_found #{Date.today}"
+      end
     end
 
     # @param dir [String, nil] DB directory
@@ -492,7 +456,8 @@ module Relaton
         FileUtils.rm_rf(fdir, secure: true)
         Util.log(
           "[relaton] WARNING: cache #{fdir}: version is obsolete and cache is "\
-            "cleared.", :warning
+            "cleared.",
+          :warning
         )
       end
       db
@@ -525,13 +490,11 @@ module Relaton
       end
 
       def global_bibliocache_name
-        Relaton.configuration.api_mode ? "cache" : "#{Dir.home}/.relaton/cache"
+        "#{Dir.home}/.relaton/cache"
       end
 
       def local_bibliocache_name(cachename)
-        return nil if Relaton.configuration.api_mode || cachename.nil?
-
-        cachename = "relaton" if cachename.empty?
+        cachename = "relaton" if cachename.nil? || cachename.empty?
         "#{cachename}/cache"
       end
     end

data/lib/relaton/db_cache.rb

@@ -1,6 +1,5 @@
 require "fileutils"
 require "timeout"
-require "relaton/storage"
 
 module Relaton
   class DbCache
@@ -11,15 +10,16 @@ module Relaton
     def initialize(dir, ext = "xml")
       @dir = dir
       @ext = ext
-      @storage = Storage.instance
-      FileUtils::mkdir_p @dir unless Relaton.configuration.api_mode
+      FileUtils::mkdir_p @dir
+      # file_version = "#{@dir}/version"
+      # set_version # unless File.exist? file_version
     end
 
     # Move caches to anothe dir
     # @param new_dir [String, nil]
     # @return [String, nil]
     def mv(new_dir)
-      return unless new_dir && @ext == "xml" && !Relaton.configuration.api_mode
+      return unless new_dir && @ext == "xml"
 
       if File.exist? new_dir
         warn "[relaton] WARNING: target directory exists \"#{new_dir}\""
@@ -32,9 +32,7 @@ module Relaton
 
     # Clear database
     def clear
-      return if Relaton.configuration.api_mode
-
-      FileUtils.rm_rf Dir.glob "#{dir}/*" if @ext == "xml" # if it's static DB
+      FileUtils.rm_rf Dir.glob "#{dir}/*" if @ext == "xml" # unless it's static DB
     end
 
     # Save item
@@ -45,10 +43,21 @@ module Relaton
         delete key
         return
       end
-      /^(?<pref>[^(]+)(?=\()/ =~ key.downcase
-      prefix_dir = "#{@dir}/#{pref}"
-      file = "#{filename(key)}.#{ext(value)}"
-      @storage.save prefix_dir, file, value
+
+      prefix_dir = "#{@dir}/#{prefix(key)}"
+      FileUtils::mkdir_p prefix_dir unless Dir.exist? prefix_dir
+      set_version prefix_dir
+      file_safe_write "#{filename(key)}.#{ext(value)}", value
+    end
+
+    # @param value [String]
+    # @return [String]
+    def ext(value)
+      case value
+      when /^not_found/ then "notfound"
+      when /^redirection/ then "redirect"
+      else @ext
+      end
     end
 
     # Read item
@@ -77,7 +86,7 @@ module Relaton
       value = self[key]
       return unless value
 
-      if value.match?(/^not_found/)
+      if value.match? /^not_found/
         value.match(/\d{4}-\d{2}-\d{2}/).to_s
       else
         doc = Nokogiri::XML value
@@ -87,21 +96,41 @@ module Relaton
 
     # Returns all items
     # @return [Array<String>]
-    def all(&block)
-      @storage.all(@dir, &block)
+    def all
+      Dir.glob("#{@dir}/**/*.{xml,yml,yaml}").sort.map do |f|
+        content = File.read(f, encoding: "utf-8")
+        block_given? ? yield(f, content) : content
+      end
     end
 
     # Delete item
     # @param key [String]
     def delete(key)
-      @storage.delete filename(key)
+      file = filename key
+      f = search_ext(file)
+      File.delete f if f
     end
 
     # Check if version of the DB match to the gem grammar hash.
     # @param fdir [String] dir pathe to flover cache
-    # @return [Boolean]
+    # @return [TrueClass, FalseClass]
     def check_version?(fdir)
-      @storage.check_version? fdir
+      version_dir = fdir + "/version"
+      return false unless File.exist? version_dir
+
+      v = File.read version_dir, encoding: "utf-8"
+      v.strip == grammar_hash(fdir)
+    end
+
+    # Set version of the DB to the gem grammar hash.
+    # @param fdir [String] dir pathe to flover cache
+    # @return [Relaton::DbCache]
+    def set_version(fdir)
+      file_version = "#{fdir}/version"
+      unless File.exist? file_version
+        file_safe_write file_version, grammar_hash(fdir)
+      end
+      self
     end
 
     # if cached reference is undated, expire it after 60 days
@@ -115,54 +144,112 @@ module Relaton
       year || Date.today - date < 60
     end
 
+    protected
+
+    # @param fdir [String] dir pathe to flover cache
+    # @return [String]
+    def grammar_hash(fdir)
+      type = fdir.split("/").last
+      Relaton::Registry.instance.by_type(type)&.grammar_hash
+    end
+
     # Reads file by a key
     #
     # @param key [String]
     # @return [String, NilClass]
     def get(key)
-      @storage.get filename(key), static: @ext == "yml"
+      file = filename key
+      return unless (f = search_ext(file))
+
+      File.read(f, encoding: "utf-8")
     end
 
     private
 
-    # @param value [String]
-    # @return [String]
-    def ext(value)
-      case value
-      when /^not_found/ then "notfound"
-      when /^redirection/ then "redirect"
-      else @ext
-      end
+    # Check if a file content is redirection
+    #
+    # @prarm value [String] file content
+    # @return [String, NilClass] redirection code or nil
+    def redirect?(value)
+      %r{redirection\s(?<code>.*)} =~ value
+      code
     end
 
     # Return item's file name
     # @param key [String]
     # @return [String]
     def filename(key)
-      prefcode = key.downcase.match(/^(?<prefix>[^(]+)\((?<code>[^)]+)/)
+      prefcode = key.downcase.match /^(?<prefix>[^\(]+)\((?<code>[^\)]+)/
       fn = if prefcode
-             "#{prefcode[:prefix]}/#{prefcode[:code].gsub(/[-:\s\/()]/, '_')
-               .squeeze('_')}"
+             "#{prefcode[:prefix]}/#{prefcode[:code].gsub(/[-:\s\/\()]/, '_').squeeze('_')}"
            else
              key.gsub(/[-:\s]/, "_")
            end
       "#{@dir}/#{fn.sub(/(,|_$)/, '')}"
     end
 
-    # Check if a file content is redirection
     #
-    # @prarm value [String] file content
-    # @return [String, NilClass] redirection code or nil
-    def redirect?(value)
-      %r{redirection\s(?<code>.*)} =~ value
-      code
+    # Checks if there is file with xml or txt extension and return filename with
+    # the extension.
+    #
+    # @param file [String]
+    # @return [String, NilClass]
+    def search_ext(file)
+      if File.exist?("#{file}.#{@ext}")
+        "#{file}.#{@ext}"
+      elsif File.exist? "#{file}.notfound"
+        "#{file}.notfound"
+      elsif File.exist? "#{file}.redirect"
+        "#{file}.redirect"
+      end
     end
 
     # Return item's subdir
     # @param key [String]
     # @return [String]
-    # def prefix(key)
-    #   key.downcase.match(/^[^(]+(?=\()/).to_s
-    # end
+    def prefix(key)
+      key.downcase.match(/^[^\(]+(?=\()/).to_s
+    end
+
+    # @param file [String]
+    # @content [String]
+    def file_safe_write(file, content)
+      File.open file, File::RDWR | File::CREAT, encoding: "UTF-8" do |f|
+        Timeout.timeout(10) { f.flock File::LOCK_EX }
+        f.write content
+      end
+    end
+
+    class << self
+      private
+
+      def global_bibliocache_name
+        "#{Dir.home}/.relaton/cache"
+      end
+
+      def local_bibliocache_name(cachename)
+        return nil if cachename.nil?
+
+        cachename = "relaton" if cachename.empty?
+        "#{cachename}/cache"
+      end
+
+      public
+
+      # Initialse and return relaton instance, with local and global cache names
+      # local_cache: local cache name; none created if nil; "relaton" created
+      # if empty global_cache: boolean to create global_cache
+      # flush_caches: flush caches
+      def init_bib_caches(opts) # rubocop:disable Metrics/CyclomaticComplexity
+        globalname = global_bibliocache_name if opts[:global_cache]
+        localname = local_bibliocache_name(opts[:local_cache])
+        localname = "relaton" if localname&.empty?
+        if opts[:flush_caches]
+          FileUtils.rm_rf globalname unless globalname.nil?
+          FileUtils.rm_rf localname unless localname.nil?
+        end
+        Relaton::Db.new(globalname, localname)
+      end
+    end
   end
 end