relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/core/hash_keys_sybolizer.rb

@@ -0,0 +1,19 @@
+module Relaton
+  module Core
+    module HashKeysSymbolizer
+      def symbolize_hash_keys(obj)
+        case obj
+        when Array
+          obj.map { |e| symbolize_hash_keys(e) }
+        when Hash
+          obj.each_with_object({}) do |(k, v), h|
+            key = k.is_a?(String) ? k.to_sym : k
+            h[key] = symbolize_hash_keys(v)
+          end
+        else
+          obj
+        end
+      end
+    end
+  end
+end

data/lib/relaton/core/hit.rb

@@ -0,0 +1,49 @@
+require "weakref"
+
+module Relaton
+  module Core
+    class Hit
+      # @return [RelatonBib::HitCollection]
+      attr_accessor :hit_collection
+
+      # @return [Array<Hash>]
+      attr_reader :hit
+
+      # @param hit [Hash]
+      # @param hit_collection [RelatonBib::HitCollection]
+      def initialize(hit, hit_collection = nil)
+        @hit            = hit
+        @hit_collection = WeakRef.new hit_collection if hit_collection
+      end
+
+      # @return [String]
+      def to_s
+        inspect
+      end
+
+      # @return [String]
+      def inspect
+        "<#{self.class}:#{format('%<id>#.14x', id: object_id << 1)} " \
+          "@reference=\"#{@hit_collection&.ref}\" " \
+          "@fetched=\"#{fetched?}\" " \
+          "@docidentifier=\"#{@hit[:code]}\">"
+      end
+
+      # @return [RelatonBib::ItemData]
+      def item
+        raise "Not implemented"
+      end
+
+      def fetched?
+        !@item.nil?
+      end
+
+      # @param opts [Hash]
+      # @option opts [Boolean] :bibdata
+      # @return [String] XML
+      def to_xml(**opts)
+        item.to_xml(**opts)
+      end
+    end
+  end
+end

data/lib/relaton/core/hit_collection.rb

@@ -0,0 +1,118 @@
+require "forwardable"
+require "nokogiri"
+require "weakref"
+require_relative "hit"
+
+module Relaton
+  module Core
+    class HitCollection
+      extend Forwardable
+
+      def_delegators :@array, :<<, :[], :first, :empty?, :any?, :size, :each, :each_slice, :reduce, :map
+
+      # @return [TrueClass, FalseClass]
+      attr_reader :fetched
+
+      # @return [String]
+      attr_reader :ref
+
+      # @return [String]
+      attr_reader :year
+
+      #
+      # @param ref [String, Pubid] reference to search
+      # @param year [String, nil] year of publication
+      #
+      def initialize(ref, year = nil)
+        @array = []
+        @ref = ref
+        @year = year
+        @fetched = false
+      end
+
+      #
+      # Fetches hits from the data source
+      #
+      # @return [self] self object
+      #
+      def fetch
+        workers = WorkersPool.new 4
+        workers.worker(&:item)
+        each do |hit|
+          workers << hit
+        end
+        workers.end
+        workers.result
+        @fetched = true
+        self
+      end
+
+      #
+      # Renders the collection as XML
+      #
+      # @param opts [Hash] options
+      # @option opts [Nokogiri::XML::Builder] :builder XML builder
+      # @option opts [Boolean] :bibdata render bibdata if true
+      # @option opts [String, Symbol] :lang language
+      #
+      # @return [String] XML representation of the collection
+      #
+      def to_xml(**opts)
+        builder = Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
+          xml.documents do
+            @array.each do |hit|
+              xml << hit.to_xml(**opts)
+            end
+          end
+        end
+        builder.to_xml
+      end
+
+      #
+      # Selects matching hits and returns a new collection
+      #
+      # @param [Proc] &block proc to select hits
+      #
+      # @return [RelatonBib::HitCollection] new hit collection
+      #
+      def select!(&block)
+        @array.select!(&block)
+        self
+      end
+
+      def select(&block)
+        array = @array.select(&block)
+        self.class.new(@ref, @year).tap do |hc|
+          new_array = array.map do |hit|
+            hit.dup.tap { |h| h.hit_collection = WeakRef.new(hc) }
+          end
+          hc.instance_variable_set(:@array, new_array)
+          hc.instance_variable_set(:@fetched, @fetched)
+        end
+      end
+
+      def reduce!(sum, &block)
+        @array = @array.reduce sum, &block
+        self
+      end
+
+      #
+      # Returns String representation of the collection
+      #
+      # @return [String] String representation of the collection
+      #
+      def to_s
+        inspect
+      end
+
+      #
+      # Returns String representation of the collection
+      #
+      # @return [String] String representation of the collection
+      #
+      def inspect
+        "<#{self.class}:#{format('%#.14x', object_id << 1)} @ref=#{@ref} @fetched=#{@fetched}>"
+      end
+    end
+  end
+end

data/lib/relaton/core/processor.rb

@@ -0,0 +1,48 @@
+module Relaton
+  module Core
+    class Processor
+      # @rerurn [Symbol]
+      attr_reader :short
+
+      # @return [String]
+      attr_reader :prefix, :idtype
+
+      # @return [Regexp]
+      attr_reader :defaultprefix
+
+      # @return [Array<String>]
+      attr_reader :datasets
+
+      def initialize
+        raise "This is an abstract class!"
+      end
+
+      def get(_code, _date, _opts)
+        raise "This is an abstract class!"
+      end
+
+      def fetch_data(_source, _opts)
+        raise "This is an abstract class!"
+      end
+
+      def from_xml(_xml)
+        raise "This is an abstract class!"
+      end
+
+      def from_yaml(_hash)
+        raise "This is an abstract class!"
+      end
+
+      def grammar_hash
+        raise "This is an abstract class!"
+      end
+
+      # Retuns default number of workers. Should be overraded by childred classes if need.
+      #
+      # @return [Integer] nuber of wokrers
+      def threads
+        10
+      end
+    end
+  end
+end

data/lib/relaton/core/workers_pool.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Core
+    # Workers poll.
+    class WorkersPool
+      attr_accessor :nb_hits
+
+      def initialize(num_workers = 2)
+        @num_workers = num_workers < 2 ? 2 : num_workers
+        @queue = SizedQueue.new(num_workers * 2)
+        @result = []
+        @nb_hits = 0
+      end
+
+      def worker(&block)
+        @threads = Array.new @num_workers do
+          Thread.new do
+            until (item = @queue.pop) == :END
+              @result << yield(item) if block
+            end
+          end
+        end
+      end
+
+      def result
+        @threads.each(&:join)
+        @result
+      end
+
+      def <<(item)
+        @queue << item
+        self
+      end
+
+      def end
+        @num_workers.times { @queue << :END }
+      end
+
+      def size
+        @result.size
+      end
+    end
+  end
+end

data/lib/relaton/core.rb

@@ -0,0 +1,9 @@
+require "relaton/logger"
+require_relative "version"
+require_relative "core/array_wrapper"
+require_relative "core/hash_keys_sybolizer"
+require_relative "core/date_parser"
+require_relative "core/processor"
+require_relative "core/data_fetcher"
+require_relative "core/hit_collection"
+require_relative "core/workers_pool"

data/lib/relaton/db/cache.rb

@@ -0,0 +1,232 @@
+require "fileutils"
+require "timeout"
+
+module Relaton
+  class Db
+    class Cache
+      # @return [String]
+      attr_reader :dir
+
+      # @param dir [String] DB directory
+      def initialize(dir, ext = "xml")
+        @dir = dir
+        @ext = ext
+        FileUtils::mkdir_p dir
+      end
+
+      # Move caches to anothe dir
+      # @param new_dir [String, nil]
+      # @return [String, nil]
+      def mv(new_dir)
+        return unless new_dir && @ext == "xml"
+
+        if File.exist? new_dir
+          Util.info "target directory exists `#{new_dir}`"
+          return
+        end
+
+        FileUtils.mv dir, new_dir
+        @dir = new_dir
+      end
+
+      # Clear database
+      def clear
+        FileUtils.rm_rf Dir.glob "#{dir}/*"
+      end
+
+      # Save item
+      # @param key [String]
+      # @param value [String] Bibitem xml serialization
+      def []=(key, value)
+        if value.nil?
+          delete key
+          return
+        end
+
+        prefix_dir = "#{@dir}/#{prefix(key)}"
+        FileUtils::mkdir_p prefix_dir
+        set_version prefix_dir
+        file_safe_write "#{filename(key)}.#{ext(value)}", value
+      end
+
+      # @param value [String]
+      # @return [String]
+      def ext(value)
+        case value
+        when /^not_found/ then "notfound"
+        when /^redirection/ then "redirect"
+        else @ext
+        end
+      end
+
+      # Read item
+      # @param key [String]
+      # @return [String]
+      def [](key)
+        value = get(key)
+        if (code = redirect_code value)
+          self[code]
+        else
+          value
+        end
+      end
+
+      #
+      # Save entry from cache of `db` to this cache.
+      #
+      # @param [String] key key of the entry
+      # @param [Relaton::Db] db database
+      #
+      def clone_entry(key, db)
+        self[key] ||= db.get(key)
+        if (code = redirect_code get(key))
+          clone_entry code, db
+        end
+      end
+
+      # Return fetched date
+      # @param key [String]
+      # @return [String]
+      def fetched(key)
+        value = self[key]
+        return unless value
+
+        if value.match?(/^not_found/)
+          value.match(/\d{4}-\d{2}-\d{2}/).to_s
+        else
+          doc = Nokogiri::XML value
+          doc.at("/bibitem/fetched|bibdata/fetched")&.text
+        end
+      end
+
+      # Returns all items
+      # @return [Array<String>]
+      def all(&block)
+        Dir.glob("#{@dir}/**/*.{xml,yml,yaml}").map do |f|
+          content = File.read(f, encoding: "utf-8")
+          block ? yield(f, content) : content
+        end
+      end
+
+      # Delete item
+      # @param key [String]
+      def delete(key)
+        file = filename key
+        f = search_ext file
+        return unless f
+
+        if File.extname(f) == ".redirect"
+          code = redirect_code get(key)
+          delete code if code
+        end
+        File.delete f
+      end
+
+      # Check if version of the DB match to the gem grammar hash.
+      # @param fdir [String] dir pathe to flover cache
+      # @return [Boolean]
+      def check_version?(fdir)
+        version_dir = "#{fdir}/version"
+        return false unless File.exist? version_dir
+
+        v = File.read version_dir, encoding: "utf-8"
+        v.strip == self.class.grammar_hash(fdir)
+      end
+
+      # if cached reference is undated, expire it after 60 days
+      # @param key [String]
+      # @param year [String]
+      def valid_entry?(key, year)
+        datestr = fetched key
+        return false unless datestr
+
+        date = Date.parse datestr
+        year || Date.today - date < 60
+      end
+
+      # Reads file by a key
+      #
+      # @param key [String]
+      # @return [String, NilClass]
+      def get(key)
+        file = filename key
+        return unless (f = search_ext(file))
+
+        File.read(f, encoding: "utf-8")
+      end
+
+      # @param fdir [String] dir pathe to flover cache
+      # @return [String]
+      def self.grammar_hash(fdir)
+        type = fdir.split("/").last
+        Registry.instance.by_type(type)&.grammar_hash
+      end
+
+      private
+
+      # @param value [String]
+      # @return [String]
+      def filename(key)
+        prefcode = key.downcase.match(/^(?<prefix>[^(]+)\((?<code>[^)]+)/)
+        fn = if prefcode
+               "#{prefcode[:prefix]}/#{prefcode[:code].gsub(/[:\s\/()]/,
+                                                            '_').squeeze('_')}"
+             else
+               key.gsub(/[-:\s]/, "_")
+             end
+        "#{@dir}/#{fn.sub(/(,|_$)/, '')}"
+      end
+
+      #
+      # Checks if there is file with xml or txt extension and return filename with
+      # the extension.
+      #
+      # @param file [String]
+      # @return [String, NilClass]
+      def search_ext(file)
+        if File.exist?("#{file}.#{@ext}")
+          "#{file}.#{@ext}"
+        elsif File.exist? "#{file}.notfound"
+          "#{file}.notfound"
+        elsif File.exist? "#{file}.redirect"
+          "#{file}.redirect"
+        end
+      end
+
+      # Set version of the DB to the gem grammar hash.
+      # @param fdir [String] dir pathe to flover cache
+      def set_version(fdir)
+        file_version = "#{fdir}/version"
+        unless File.exist? file_version
+          file_safe_write file_version, self.class.grammar_hash(fdir)
+        end
+      end
+
+      # Return item's file name
+      # @param key [String]
+      # @return [String]
+      def prefix(key)
+        key.downcase.match(/^[^(]+(?=\()/).to_s
+      end
+
+      # Check if a file content is redirection
+      #
+      # @prarm value [String] file content
+      # @return [String, NilClass] redirection code or nil
+      def redirect_code(value)
+        %r{redirection\s(?<code>.*)} =~ value
+        code
+      end
+
+      # @param file [String]
+      # @content [String]
+      def file_safe_write(file, content)
+        File.open file, File::RDWR | File::CREAT, encoding: "UTF-8" do |f|
+          Timeout.timeout(10) { f.flock File::LOCK_EX }
+          f.write content
+          f.flock File::LOCK_UN
+        end
+      end
+    end
+  end
+end

data/lib/relaton/db/config.rb

@@ -0,0 +1,24 @@
+module Relaton
+  class Db
+    module Config
+      def configure
+        yield configuration if block_given?
+      end
+
+      def configuration
+        @configuration ||= Configuration.new
+      end
+    end
+
+    class Configuration
+      attr_accessor :use_api, :api_host
+
+      def initialize
+        @use_api = false
+        @api_host = "https://api.relaton.org"
+      end
+    end
+
+    extend Config
+  end
+end

data/lib/relaton/db/registry.rb

@@ -0,0 +1,119 @@
+require "singleton"
+
+module Relaton
+  class Db
+    class Registry
+      SUPPORTED_GEMS = %w[
+        relaton/gb relaton/iec relaton/ietf relaton/iso
+        relaton/itu relaton/nist relaton/ogc relaton/calconnect
+        relaton/omg relaton/un relaton/w3c relaton/ieee
+        relaton/iho relaton/bipm relaton/ecma relaton/cie
+        relaton/bsi relaton/cen relaton/iana relaton/3gpp
+        relaton/oasis relaton/doi relaton/jis relaton/xsf
+        relaton/ccsds relaton/etsi relaton/isbn relaton/plateau
+        relaton/oiml
+      ].freeze
+
+      include Singleton
+
+      attr_reader :processors
+
+      def initialize
+        @processors = {}
+        register_gems
+      end
+
+      def register_gems
+        SUPPORTED_GEMS.each do |b|
+          # Load ONLY the lightweight processor file; its heavy flavor code is
+          # lazy-required inside the processor's methods. This keeps
+          # `require "relaton"` from eagerly loading every flavor at start.
+          require "#{b}/processor"
+          register Kernel.const_get "#{gem_to_module_path(b)}::Processor"
+        rescue LoadError => e
+          Util.error "backend #{b} not present\n" \
+                     "#{e.message}\n#{e.backtrace.join "\n"}"
+        end
+      end
+
+      def register(processor)
+        raise Error unless processor < Core::Processor
+
+        p = processor.new
+        return if processors[p.short]
+
+        Util.debug("processor \"#{p.short}\" registered")
+        processors[p.short] = p
+      end
+
+      def find_processor(short)
+        processors[short.to_sym]
+      end
+
+      # @return [Array<Symbol>]
+      def supported_processors
+        processors.keys
+      end
+
+      #
+      # Search a rpocessos by dataset name
+      #
+      # @param [String] dataset
+      #
+      # @return [Relaton::Core::Processor, nil]
+      #
+      def find_processor_by_dataset(dataset)
+        processors.values.detect { |p| p.datasets&.include? dataset }
+      end
+
+      #
+      # Find processor by type
+      #
+      # @param type [String]
+      # @return [Relaton::Core::Processor]
+      def by_type(type)
+        processors.values.detect { |v| v.prefix == type&.upcase }
+      end
+
+      def [](stdclass)
+        processors[stdclass]
+      end
+
+      #
+      # Find processor by reference or prefix
+      #
+      # @param [String] ref reference or prefix
+      #
+      # @return [Relaton::Core::Processor] processor
+      #
+      def processor_by_ref(ref)
+        processors[class_by_ref(ref)]
+      end
+
+      #
+      # Find processor by refernce or prefix
+      #
+      # @param ref [String] reference or prefix
+      #
+      # @return [Symbol, nil] standard class name
+      #
+      def class_by_ref(ref)
+        ref = Regexp.last_match(1) if ref =~ /^\w+\((.*)\)$/
+        @processors.each do |class_name, processor|
+          return class_name if /^(urn:)?#{processor.prefix}\b/i.match?(ref) ||
+            processor.defaultprefix.match(ref)
+        end
+        Util.info "`#{ref}` does not have a recognised prefix", key: ref
+        nil
+      end
+
+      private
+
+      def gem_to_module_path(gem_name)
+        gem_name.split("/").map do |part|
+          part.capitalize.sub("3gpp", "ThreeGpp")
+        end.join("::")
+      end
+    end
+  end
+end

data/lib/relaton/db/util.rb

@@ -0,0 +1,9 @@
+module Relaton
+  class Db
+    module Util
+      extend Relaton::Bib::Util
+
+      PROGNAME = "relaton-db".freeze
+    end
+  end
+end

data/lib/relaton/db/workers_pool.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Relaton
+  class Db
+    # Workers poll.
+    class WorkersPool
+      def initialize(workers = 2, &)
+        @queue = SizedQueue.new(workers * 2)
+        @threads = Array.new workers do
+          Thread.new do
+            while item = @queue.pop; yield(item) end
+          end
+        end
+      end
+
+      def <<(item)
+        @queue << item
+        self
+      end
+    end
+  end
+end