relaton 2.1.0 → 2.2.0.pre.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5774763b35647935239061ed5da1c618f8c2c84b31adbf83afb56aa7a920da10
-  data.tar.gz: 612177a17fbbca7af8d5a32d5a408ef82e6ce7a00a1f37e9c6e6302976d6dd70
+  metadata.gz: 7836d4ee81be911e0bf0a95a3b9154895cfef800e918443a54ed73d89623b268
+  data.tar.gz: ac550984b45d4c4b558a277c23c4a3163903bdb070760eee2395f070b75b8980
 SHA512:
-  metadata.gz: 3284e5fc96ed6d0b4035c093d8bef53dcda4225a9477fb137610829f6676674702020541cba00ba8cc41dafcd6548eb8eafbcf6fc2e6a800b1ba844d38e71e91
-  data.tar.gz: 97acc11738d7b016522b9b145e056c3b9fe934307a94e3d83ea5d78c2259a55260f94c49dcd5fb928bf184373fa79a37708900326323c29fc4388522da7fa88b
+  metadata.gz: 7fd1668edf1c684c26e3cbf3d2177654b1b48d103290101e457a9811c49dcf6dbb85fe48750985527c545f5a3e30b627a2bff5c2d39ad87e28358528d434c891
+  data.tar.gz: fef35f22ec455e5c1094ba2016355b64f0aaa60fb9ce569dc4e4a19ea7c8c3aa0817ea94dd7ba74c039829ecf18f6d01f4541c59734c0183a63c8b0c7632d204

data/.github/workflows/rake.yml

@@ -4,7 +4,7 @@ name: rake
 
 on:
   push:
-    branches: [ master, main ]
+    branches: [ master, main, lutaml-integration ]
     tags: [ v* ]
   pull_request:
   workflow_dispatch:

data/CLAUDE.md

@@ -63,6 +63,14 @@ Thread pool for `fetch_async`. Default 10 threads per processor, overridable via
 - Tests create `testcache`/`testcache2` directories and clean them in `before(:each)`
 - Cache-related tests need `<fetched>` elements in XML for `valid_entry?` to return true
 - Integration tests in `spec/relaton_spec.rb`; unit tests mirror `lib/` structure under `spec/relaton/`
+- **ISO lookups are stubbed, not cassette-recorded.** Flavor gems (relaton-iso/iec/nist)
+  fetch a large live `index-v2` and deserialize every id through a pinned pubid build,
+  so a single drifted id in the live index makes the whole index unparseable and ISO
+  lookups return `nil`. Umbrella specs therefore stub `Relaton::Iso::Bibliography.get`
+  (and other flavors' `.get`) to return hand-built `ItemData` — the umbrella's job is to
+  test `Db` orchestration (`combine_doc`, caching, api fallback), not relaton-iso's index.
+  Build stub items with the `docidentifier:` key (not `docid:`) so the id survives the
+  cache XML round-trip. Don't reintroduce a live-index cassette for these.
 
 ## Style
 

data/Gemfile

@@ -3,6 +3,14 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in gemspec
 gemspec
 
+# Use local monorepo sibling gems where available.
+Dir["../*/"].each do |dir|
+  name = File.basename(dir)
+  next if name == File.basename(__dir__)
+  next unless File.exist?(File.join(dir, "#{name}.gemspec"))
+  gem name, path: dir
+end
+
 
 
 gem "byebug"

data/lib/relaton/db/cache.rb

@@ -0,0 +1,232 @@
+require "fileutils"
+require "timeout"
+
+module Relaton
+  class Db
+    class Cache
+      # @return [String]
+      attr_reader :dir
+
+      # @param dir [String] DB directory
+      def initialize(dir, ext = "xml")
+        @dir = dir
+        @ext = ext
+        FileUtils::mkdir_p dir
+      end
+
+      # Move caches to anothe dir
+      # @param new_dir [String, nil]
+      # @return [String, nil]
+      def mv(new_dir)
+        return unless new_dir && @ext == "xml"
+
+        if File.exist? new_dir
+          Util.info "target directory exists `#{new_dir}`"
+          return
+        end
+
+        FileUtils.mv dir, new_dir
+        @dir = new_dir
+      end
+
+      # Clear database
+      def clear
+        FileUtils.rm_rf Dir.glob "#{dir}/*"
+      end
+
+      # Save item
+      # @param key [String]
+      # @param value [String] Bibitem xml serialization
+      def []=(key, value)
+        if value.nil?
+          delete key
+          return
+        end
+
+        prefix_dir = "#{@dir}/#{prefix(key)}"
+        FileUtils::mkdir_p prefix_dir
+        set_version prefix_dir
+        file_safe_write "#{filename(key)}.#{ext(value)}", value
+      end
+
+      # @param value [String]
+      # @return [String]
+      def ext(value)
+        case value
+        when /^not_found/ then "notfound"
+        when /^redirection/ then "redirect"
+        else @ext
+        end
+      end
+
+      # Read item
+      # @param key [String]
+      # @return [String]
+      def [](key)
+        value = get(key)
+        if (code = redirect_code value)
+          self[code]
+        else
+          value
+        end
+      end
+
+      #
+      # Save entry from cache of `db` to this cache.
+      #
+      # @param [String] key key of the entry
+      # @param [Relaton::Db] db database
+      #
+      def clone_entry(key, db)
+        self[key] ||= db.get(key)
+        if (code = redirect_code get(key))
+          clone_entry code, db
+        end
+      end
+
+      # Return fetched date
+      # @param key [String]
+      # @return [String]
+      def fetched(key)
+        value = self[key]
+        return unless value
+
+        if value.match?(/^not_found/)
+          value.match(/\d{4}-\d{2}-\d{2}/).to_s
+        else
+          doc = Nokogiri::XML value
+          doc.at("/bibitem/fetched|bibdata/fetched")&.text
+        end
+      end
+
+      # Returns all items
+      # @return [Array<String>]
+      def all(&block)
+        Dir.glob("#{@dir}/**/*.{xml,yml,yaml}").map do |f|
+          content = File.read(f, encoding: "utf-8")
+          block ? yield(f, content) : content
+        end
+      end
+
+      # Delete item
+      # @param key [String]
+      def delete(key)
+        file = filename key
+        f = search_ext file
+        return unless f
+
+        if File.extname(f) == ".redirect"
+          code = redirect_code get(key)
+          delete code if code
+        end
+        File.delete f
+      end
+
+      # Check if version of the DB match to the gem grammar hash.
+      # @param fdir [String] dir pathe to flover cache
+      # @return [Boolean]
+      def check_version?(fdir)
+        version_dir = "#{fdir}/version"
+        return false unless File.exist? version_dir
+
+        v = File.read version_dir, encoding: "utf-8"
+        v.strip == self.class.grammar_hash(fdir)
+      end
+
+      # if cached reference is undated, expire it after 60 days
+      # @param key [String]
+      # @param year [String]
+      def valid_entry?(key, year)
+        datestr = fetched key
+        return false unless datestr
+
+        date = Date.parse datestr
+        year || Date.today - date < 60
+      end
+
+      # Reads file by a key
+      #
+      # @param key [String]
+      # @return [String, NilClass]
+      def get(key)
+        file = filename key
+        return unless (f = search_ext(file))
+
+        File.read(f, encoding: "utf-8")
+      end
+
+      # @param fdir [String] dir pathe to flover cache
+      # @return [String]
+      def self.grammar_hash(fdir)
+        type = fdir.split("/").last
+        Registry.instance.by_type(type)&.grammar_hash
+      end
+
+      private
+
+      # @param value [String]
+      # @return [String]
+      def filename(key)
+        prefcode = key.downcase.match(/^(?<prefix>[^(]+)\((?<code>[^)]+)/)
+        fn = if prefcode
+               "#{prefcode[:prefix]}/#{prefcode[:code].gsub(/[:\s\/()]/,
+                                                            '_').squeeze('_')}"
+             else
+               key.gsub(/[-:\s]/, "_")
+             end
+        "#{@dir}/#{fn.sub(/(,|_$)/, '')}"
+      end
+
+      #
+      # Checks if there is file with xml or txt extension and return filename with
+      # the extension.
+      #
+      # @param file [String]
+      # @return [String, NilClass]
+      def search_ext(file)
+        if File.exist?("#{file}.#{@ext}")
+          "#{file}.#{@ext}"
+        elsif File.exist? "#{file}.notfound"
+          "#{file}.notfound"
+        elsif File.exist? "#{file}.redirect"
+          "#{file}.redirect"
+        end
+      end
+
+      # Set version of the DB to the gem grammar hash.
+      # @param fdir [String] dir pathe to flover cache
+      def set_version(fdir)
+        file_version = "#{fdir}/version"
+        unless File.exist? file_version
+          file_safe_write file_version, self.class.grammar_hash(fdir)
+        end
+      end
+
+      # Return item's file name
+      # @param key [String]
+      # @return [String]
+      def prefix(key)
+        key.downcase.match(/^[^(]+(?=\()/).to_s
+      end
+
+      # Check if a file content is redirection
+      #
+      # @prarm value [String] file content
+      # @return [String, NilClass] redirection code or nil
+      def redirect_code(value)
+        %r{redirection\s(?<code>.*)} =~ value
+        code
+      end
+
+      # @param file [String]
+      # @content [String]
+      def file_safe_write(file, content)
+        File.open file, File::RDWR | File::CREAT, encoding: "UTF-8" do |f|
+          Timeout.timeout(10) { f.flock File::LOCK_EX }
+          f.write content
+          f.flock File::LOCK_UN
+        end
+      end
+    end
+  end
+end

data/lib/relaton/db/config.rb

@@ -0,0 +1,24 @@
+module Relaton
+  class Db
+    module Config
+      def configure
+        yield configuration if block_given?
+      end
+
+      def configuration
+        @configuration ||= Configuration.new
+      end
+    end
+
+    class Configuration
+      attr_accessor :use_api, :api_host
+
+      def initialize
+        @use_api = false
+        @api_host = "https://api.relaton.org"
+      end
+    end
+
+    extend Config
+  end
+end

data/lib/relaton/db/registry.rb

@@ -0,0 +1,119 @@
+require "singleton"
+
+module Relaton
+  class Db
+    class Registry
+      SUPPORTED_GEMS = %w[
+        relaton/gb relaton/iec relaton/ietf relaton/iso
+        relaton/itu relaton/nist relaton/ogc relaton/calconnect
+        relaton/omg relaton/un relaton/w3c relaton/ieee
+        relaton/iho relaton/bipm relaton/ecma relaton/cie
+        relaton/bsi relaton/cen relaton/iana relaton/3gpp
+        relaton/oasis relaton/doi relaton/jis relaton/xsf
+        relaton/ccsds relaton/etsi relaton/isbn relaton/plateau
+      ].freeze
+
+      include Singleton
+
+      attr_reader :processors
+
+      def initialize
+        @processors = {}
+        register_gems
+      end
+
+      def register_gems
+        SUPPORTED_GEMS.each do |b|
+          # Require the gem's top-level file so its internal modules
+          # (e.g. Relaton::Iso::Util) load in the right order before we
+          # poke at its Processor class.
+          require b
+          require "#{b}/processor"
+          register Kernel.const_get "#{gem_to_module_path(b)}::Processor"
+        rescue LoadError => e
+          Util.error "backend #{b} not present\n" \
+                     "#{e.message}\n#{e.backtrace.join "\n"}"
+        end
+      end
+
+      def register(processor)
+        raise Error unless processor < Core::Processor
+
+        p = processor.new
+        return if processors[p.short]
+
+        Util.debug("processor \"#{p.short}\" registered")
+        processors[p.short] = p
+      end
+
+      def find_processor(short)
+        processors[short.to_sym]
+      end
+
+      # @return [Array<Symbol>]
+      def supported_processors
+        processors.keys
+      end
+
+      #
+      # Search a rpocessos by dataset name
+      #
+      # @param [String] dataset
+      #
+      # @return [Relaton::Core::Processor, nil]
+      #
+      def find_processor_by_dataset(dataset)
+        processors.values.detect { |p| p.datasets&.include? dataset }
+      end
+
+      #
+      # Find processor by type
+      #
+      # @param type [String]
+      # @return [Relaton::Core::Processor]
+      def by_type(type)
+        processors.values.detect { |v| v.prefix == type&.upcase }
+      end
+
+      def [](stdclass)
+        processors[stdclass]
+      end
+
+      #
+      # Find processor by reference or prefix
+      #
+      # @param [String] ref reference or prefix
+      #
+      # @return [Relaton::Core::Processor] processor
+      #
+      def processor_by_ref(ref)
+        processors[class_by_ref(ref)]
+      end
+
+      #
+      # Find processor by refernce or prefix
+      #
+      # @param ref [String] reference or prefix
+      #
+      # @return [Symbol, nil] standard class name
+      #
+      def class_by_ref(ref)
+        ref = Regexp.last_match(1) if ref =~ /^\w+\((.*)\)$/
+        @processors.each do |class_name, processor|
+          return class_name if /^(urn:)?#{processor.prefix}\b/i.match?(ref) ||
+            processor.defaultprefix.match(ref)
+        end
+        Util.info "`#{ref}` does not have a recognised prefix", key: ref
+        nil
+      end
+
+      private
+
+      def gem_to_module_path(gem_name)
+        gem_name.split("/").map do |part|
+          part.capitalize.sub("3gpp", "ThreeGpp")
+        end.join("::")
+      end
+    end
+  end
+end

data/lib/relaton/db/util.rb

@@ -0,0 +1,9 @@
+module Relaton
+  class Db
+    module Util
+      extend Relaton::Bib::Util
+
+      PROGNAME = "relaton-db".freeze
+    end
+  end
+end

data/lib/relaton/db/version.rb

@@ -0,0 +1,5 @@
+module Relaton
+  class Db
+    VERSION = "2.2.0.pre.alpha.1".freeze
+  end
+end

data/lib/relaton/db/workers_pool.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Relaton
+  class Db
+    # Workers poll.
+    class WorkersPool
+      def initialize(workers = 2, &)
+        @queue = SizedQueue.new(workers * 2)
+        @threads = Array.new workers do
+          Thread.new do
+            while item = @queue.pop; yield(item) end
+          end
+        end
+      end
+
+      def <<(item)
+        @queue << item
+        self
+      end
+    end
+  end
+end