relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/iho/structured_identifier.rb

@@ -0,0 +1,20 @@
+module Relaton
+  module Iho
+    class StructuredIdentifier < Lutaml::Model::Serializable
+      attribute :docnumber, :string
+      attribute :part, :string
+      attribute :annexid, :string
+      attribute :appendixid, :string
+      attribute :supplementid, :string
+
+      xml do
+        root "structuredidentifier"
+        map_element "docnumber", to: :docnumber
+        map_element "part", to: :part
+        map_element "annexid", to: :annexid
+        map_element "appendixid", to: :appendixid
+        map_element "supplementid", to: :supplementid
+      end
+    end
+  end
+end

data/lib/relaton/iho/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Iho
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-iho".freeze
+    end
+  end
+end

data/lib/relaton/iho.rb

@@ -0,0 +1,27 @@
+require "relaton/bib"
+require "relaton/index"
+require "pubid"
+require_relative "version"
+require_relative "iho/util"
+require_relative "iho/docidentifier"
+require_relative "iho/item"
+require_relative "iho/bibitem"
+require_relative "iho/bibdata"
+require_relative "iho/bibliography"
+
+module Relaton
+  module Iho
+    INDEXFILE = "index-v3".freeze
+
+    class Error < StandardError; end
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/index/config.rb

@@ -0,0 +1,52 @@
+module Relaton
+  module Index
+    #
+    # Configuration class for Relaton::Index
+    #
+    class Config
+      attr_reader :storage, :storage_dir, :filename
+
+      #
+      # Set default values
+      #
+      def initialize
+        @storage = FileStorage
+        @storage_dir = Dir.home
+        @filename = "index.yaml"
+      end
+
+      #
+      # Set storage
+      #
+      # @param [#ctime, #read, #write] storage storage object
+      #
+      # @return [void]
+      #
+      def storage=(storage)
+        @storage = storage
+      end
+
+      #
+      # Set storage directory
+      #
+      # @param [String] dir storage directory
+      #
+      # @return [void]
+      #
+      def storage_dir=(dir)
+        @storage_dir = dir
+      end
+
+      #
+      # Set filename
+      #
+      # @param [String] filename filename
+      #
+      # @return [void]
+      #
+      def filename=(filename)
+        @filename = filename
+      end
+    end
+  end
+end

data/lib/relaton/index/file_io.rb

@@ -0,0 +1,305 @@
+module Relaton
+  module Index
+    #
+    # File IO class is used to read and write index files.
+    # In searh mode url is used to fetch index from external repository and save it to storage.
+    # In index mode url should be nil.
+    #
+    class FileIO
+      include IdNumber
+
+      # Raised internally when a deserialized id cannot be parsed or is not
+      # understood by the pubid class; `#load_index` rescues it to trigger the
+      # wrong-structure handling (re-download, or stop and log).
+      class InvalidIndexError < StandardError; end
+
+      attr_reader :url, :pubid_class
+      attr_accessor :sorted
+
+      @@file_locks = {}
+      @@file_locks_mutex = Mutex.new
+
+      #
+      # Initialize FileIO
+      #
+      # @param [String] dir falvor specific local directory in ~/.relaton to store index
+      # @param [String, Boolean, nil] url
+      #   if String then the URL is used to fetch an index from a Git repository
+      #     and save it to the storage (if not exists, or older than 24 hours)
+      #   if true then the index is read from the storage (used to remove index file)
+      #   if nil then the fiename is used to read and write file (used to create indes in GH actions)
+      # @param [Pubid::Identifier] pubid class for deserialization
+      #
+      # `id_keys` is accepted for backward compatibility but no longer used: the
+      # index format is now validated by round-tripping a sample of ids through
+      # the pubid class (see #check_serialization), which understands the pubid
+      # v2 (lutaml) `_type` serialization that the old key-allowlist could not.
+      def initialize(dir, url, filename, _id_keys = nil, pubid_class = nil)
+        @dir = dir
+        @url = url
+        @filename = filename
+        @pubid_class = pubid_class
+        @sorted = false
+      end
+
+      #
+      # If url is String, check if index file exists and is not older than 24
+      #   hours. If not, fetch index from external repository and save it to
+      #   storage.
+      # If url is true, read index from path to local file.
+      # If url is nil, read index from filename.
+      #
+      # @return [Array<Hash>] index
+      #
+      def read
+        case url
+        when String
+          with_file_lock do
+            check_file || fetch_and_save
+          end
+        else
+          read_file || []
+        end
+      end
+
+      def file
+        @file ||= url ? path_to_local_file : @filename
+      end
+
+      #
+      # Create path to local file
+      #
+      # @return [<Type>] <description>
+      #
+      def path_to_local_file
+        File.join(Index.config.storage_dir, ".relaton", @dir, @filename)
+      end
+
+      #
+      # Check if index file exists and is not older than 24 hours
+      #
+      # @return [Array<Hash>, nil] index or nil
+      #
+      def check_file
+        ctime = Index.config.storage.ctime(file)
+        return unless ctime && ctime > Time.now - 86400
+
+        read_file
+      end
+
+      #
+      # Check if index has correct format
+      #
+      # @param [Array<Hash>] index index to check
+      #
+      # @return [Boolean] <description>
+      #
+      # Structural check only. Per-id serialization is validated during
+      # deserialization (see #deserialize_id), which reuses the `from_hash` the
+      # index load performs anyway, so every row is checked at no extra parse
+      # cost.
+      def check_format(index)
+        check_basic_format(index)
+      end
+
+      def check_basic_format(index)
+        return false unless index.is_a? Array
+
+        keys = %i[file id]
+        index.all? { |item| item.respond_to?(:keys) && item.keys.sort == keys }
+      end
+
+      # An id is supported when `from_hash` either resolves it to a concrete
+      # type (a subclass — the polymorphic `_type` matched) or round-trips
+      # losslessly through `to_hash`. The subclass clause covers valid entries
+      # pubid cannot fully rebuild on re-serialize (e.g. ISO directives drop a
+      # redundant subgroup number); the round-trip clause covers pubid classes
+      # without a subclass hierarchy. A wrong-format/garbled id satisfies
+      # neither: it falls back to the bare base class and fails to round-trip.
+      def id_supported?(obj, raw)
+        # A concrete subtype means pubid recognized the `_type`; accept without
+        # round-tripping. This both skips the false positive for valid-but-lossy
+        # types (e.g. ISO directives) and avoids the costly hash compare for the
+        # ~all rows that resolve to a subtype (it would otherwise add ~33%).
+        return true unless obj.instance_of?(@pubid_class)
+
+        normalize(obj.to_hash) == normalize(raw)
+      rescue StandardError
+        false
+      end
+
+      # Stringify hash keys and scalar values so the comparison ignores YAML
+      # scalar typing (e.g. 1 vs "1") and string/symbol key differences, while
+      # still detecting dropped/added keys or genuinely changed values.
+      def normalize(value)
+        case value
+        when Hash then value.to_h { |k, v| [k.to_s, normalize(v)] }
+        when Array then value.map { |v| normalize(v) }
+        when nil then nil
+        else value.to_s
+        end
+      end
+
+      #
+      # Read index from storage
+      #
+      # @return [Array<Hash>] index
+      #
+      def read_file
+        yaml = Index.config.storage.read(file)
+        return unless yaml
+
+        load_index(yaml) || []
+      end
+
+      # Deserialize and sort by the same narrowing key Type#search bsearches
+      # on, so binary search always has a consistent total order. The published
+      # index is only approximately sorted (generated under pubid 1.x base
+      # semantics); merely detecting sortedness left bsearch disabled and every
+      # search a full O(n) scan. Sorting here is one-time per load.
+      def deserialize_pubid(index)
+        return index unless @pubid_class
+
+        deserialized = index.map do |r|
+          { id: deserialize_id(r[:id]), file: r[:file] }
+        end
+        warn_unless_sorted(deserialized)
+        deserialized.sort_by! { |r| get_id_number(r[:id]) }
+        @sorted = true
+        deserialized
+      end
+
+      # Deserialize one id and verify pubid understands it. Reuses the
+      # `from_hash` deserialization the load performs anyway, so validating every
+      # row costs only the `to_hash`/compare for ids that need the round-trip
+      # clause. Raises InvalidIndexError when an id cannot be parsed or is
+      # unsupported, so `#load_index` rejects (and re-downloads) the whole index.
+      def deserialize_id(raw)
+        obj = @pubid_class.from_hash(raw)
+      rescue StandardError => e
+        raise InvalidIndexError, "cannot parse id #{raw.inspect}: #{e.message}"
+      else
+        return obj if id_supported?(obj, raw)
+
+        raise InvalidIndexError, "unsupported id #{raw.inspect}"
+      end
+
+      # Log when the loaded index is not already in get_id_number order, so the
+      # in-memory sort above (and the underlying not-sorted index file) is
+      # visible. Stops at the first out-of-order pair.
+      def warn_unless_sorted(index)
+        prev = nil
+        index.each do |r|
+          num = get_id_number(r[:id])
+          if prev && prev > num
+            Util.warn "Index file `#{file}` is not sorted by id number; " \
+                      "sorting #{index.size} entries in memory.", progname
+            return
+          end
+          prev = num
+        end
+      end
+
+      def warn_local_index_error(reason)
+        Util.info "#{reason} file `#{file}`", progname
+        if url.is_a? String
+          Util.info "Considering `#{file}` file corrupt, re-downloading from `#{url}`", progname
+        else
+          Util.info "Considering `#{file}` file corrupt, removing it.", progname
+          remove
+        end
+      end
+
+      def progname
+        @progname ||= "relaton-#{@dir}"
+      end
+
+      def load_index(yaml, save = false)
+        index = YAML.safe_load(yaml, permitted_classes: [Symbol])
+        save index if save
+        return deserialize_pubid(index) if check_format(index)
+
+        report_invalid_index(save, "Wrong structure of")
+      rescue Psych::SyntaxError
+        report_invalid_index(save, "YAML parsing error when reading")
+      rescue InvalidIndexError
+        report_invalid_index(save, "Wrong structure of")
+      end
+
+      def report_invalid_index(save, reason)
+        if save
+          warn_remote_index_error reason
+        else
+          warn_local_index_error reason
+        end
+      end
+
+      #
+      # Fetch index from external repository and save it to storage
+      #
+      # @return [Array<Hash>] index
+      #
+      def fetch_and_save
+        uri = URI.parse(url)
+        body = Net::HTTP.get(uri)
+        yaml = nil
+        Zip::File.open_buffer(body) do |zip|
+          entry = zip.entries.first
+          yaml = entry.get_input_stream.read
+        end
+        Util.info "Downloaded index from `#{url}`", progname
+        load_index(yaml, true)
+      end
+
+      def warn_remote_index_error(reason)
+        Util.info "#{reason} newly downloaded file `#{file}` at `#{url}`, " \
+             "the remote index seems to be invalid. Please report this " \
+             "issue at https://github.com/relaton/relaton-cli.", progname
+      end
+
+      #
+      # Save index to storage
+      #
+      # @param [Array<Hash>] index index to save
+      #
+      # @return [void]
+      #
+      def save(index)
+        yaml = sort_structured_index(index).map do |item|
+          item.transform_values do |value|
+            @pubid_class && value.is_a?(@pubid_class) ? value.to_hash : value
+          end
+        end.to_yaml
+        Index.config.storage.write file, yaml
+      end
+
+      def sort_structured_index(index)
+        if @pubid_class && index.first&.dig(:id).is_a?(@pubid_class)
+          index.sort_by { |item| get_id_number item[:id] }
+        else
+          index
+        end
+      end
+
+      #
+      # Remove index file from storage
+      #
+      # @return [Array]
+      #
+      def remove
+        Index.config.storage.remove file
+        []
+      end
+
+      private
+
+      def with_file_lock(&)
+        @@file_locks_mutex.synchronize do
+          @@file_locks[file] ||= Mutex.new
+        end
+
+        @@file_locks[file].synchronize(&)
+      end
+    end
+  end
+end

data/lib/relaton/index/file_storage.rb

@@ -0,0 +1,61 @@
+module Relaton
+  module Index
+    #
+    # File storage module contains methods to read and write files
+    #
+    module FileStorage
+      #
+      # Return file creation time
+      #
+      # @param [String] file file path
+      #
+      # @return [Time, nil] file creation time or nil if file does not exist
+      #
+      def ctime(file)
+        File.exist?(file) && File.ctime(file)
+      end
+
+      #
+      # Read file
+      #
+      # @param [String] file file path
+      #
+      # @return [String, nil] file content or nil if file does not exist
+      #
+      def read(file)
+        return unless File.exist?(file)
+
+        File.read file, encoding: "UTF-8"
+      end
+
+      #
+      # Write file
+      #
+      # @param [String] file file path
+      # @param [String] data content to write
+      #
+      # @return [void]
+      #
+      def write(file, data)
+        dir = File.dirname file
+        FileUtils.mkdir_p dir
+        File.write file, data, encoding: "UTF-8"
+      end
+
+      #
+      # Remove file
+      #
+      # @param [String] file file path
+      #
+      # @return [void]
+      #
+      def remove(file)
+        return unless File.exist? file
+
+        File.delete file
+      end
+
+      extend self
+    end
+  end
+end

data/lib/relaton/index/id_number.rb

@@ -0,0 +1,30 @@
+module Relaton
+  module Index
+    # Shared narrowing/sort key for structured (pubid) index ids. Type uses it
+    # for binary-search narrowing; FileIO uses it to sort the index and detect
+    # sortedness. The two MUST agree, so the rule lives in one place.
+    module IdNumber
+      # One-level narrowing key: a supplement/amendment is filed under its
+      # immediate parent's number, everything else under its own number.
+      #
+      # Pubid 2.x exposes the parent via `.base_identifier` (the LutaML
+      # `Pubid::Iso::Identifiers::*` classes that relaton loads at runtime). A
+      # standalone `require "pubid-iso"` can instead surface the legacy
+      # `Pubid::Iso::Identifier::*` classes, which use `.base`; we accept either
+      # so the key is stable in both load orders. The wrong accessor silently
+      # falls through to the row's own number and breaks bsearch narrowing.
+      def get_id_number(id)
+        base = id_base(id)
+        ((base && base.number) || id.number).to_s
+      end
+
+      def id_base(id)
+        if id.respond_to?(:base_identifier) && id.base_identifier
+          id.base_identifier
+        elsif id.respond_to?(:base) && id.base
+          id.base
+        end
+      end
+    end
+  end
+end

data/lib/relaton/index/pool.rb

@@ -0,0 +1,41 @@
+module Relaton
+  module Index
+    #
+    # Pool of indexes
+    #
+    class Pool
+      def initialize
+        @pool = {}
+      end
+
+      #
+      # Return index by type, create if not exists
+      #
+      # @param [String] type <description>
+      # @param [String, nil] url external URL to index, used to fetch index for searching files
+      # @param [String, nil] file output file name
+      # @param [Array<Symbol>, nil] id_keys keys to check if index is correct
+      #
+      # @return [Relaton::Index::Type] typed index
+      #
+      def type(type, **args)
+        if @pool[type.upcase.to_sym]&.actual?(**args)
+          @pool[type.upcase.to_sym]
+        else
+          @pool[type.upcase.to_sym] = Type.new(type, args[:url], args[:file], args[:id_keys], args[:pubid_class])
+        end
+      end
+
+      #
+      # Remove index by type from pool
+      #
+      # @param [String] type index type
+      #
+      # @return [void]
+      #
+      def remove(type)
+        @pool.delete type.upcase.to_sym
+      end
+    end
+  end
+end

data/lib/relaton/index/type.rb

@@ -0,0 +1,159 @@
+module Relaton
+  module Index
+    #
+    # Relaton::Index::Type is a class for indexing Relaton files.
+    #
+    class Type
+      include IdNumber
+
+      #
+      # Initialize a new Relaton::Index::Type object
+      #
+      # @param [String, Symbol] type type of index (ISO, IEC, etc.)
+      # @param [String, nil] url external URL to index, used to fetch index for searching files
+      # @param [String, nil] file output file name
+      # @param [Array<Symbol>] id_keys keys of identifier to be used for sorting index
+      #   format of index file is checked if id_keys all is provided at least in one of the IDs
+      # @param [Pubid::Identifier, nil] pubid class for deserialization
+      #
+      def initialize(type, url = nil, file = nil, id_keys = nil, pubid_class = nil) # rubocop:disable Metrics/ParameterLists
+        @file = file
+        filename = file || Index.config.filename
+        @file_io = FileIO.new type.to_s.downcase, url, filename, id_keys, pubid_class
+      end
+
+      def index
+        @index ||= @file_io.read
+      end
+
+      #
+      # Check if index is actual. If url or file is given, check if it is equal to
+      # index url or file.
+      #
+      # @param [Hash] **args arguments
+      # @option args [String, nil] :url external URL to index, used to fetch index for searching files
+      # @option args [String, nil] :file output file name
+      #
+      # @return [Boolean] true if index is actual, false otherwise
+      #
+      def actual?(**args)
+        (!args.key?(:url) || args[:url] == @file_io.url) && (!args.key?(:file) || args[:file] == @file)
+      end
+
+      #
+      # Add or update index item
+      #
+      # @param [Pubid::Identifier] id document ID
+      # @param [String] file file name of the document
+      #
+      # @return [void]
+      #
+      def add_or_update(id, file)
+        key = id.to_s
+        item = id_lookup[key]
+        if item
+          item[:file] = file
+        else
+          new_item = { id: id, file: file }
+          index << new_item
+          id_lookup[key] = new_item
+          @file_io.sorted = false
+        end
+      end
+
+      #
+      # Search index for a given ID
+      #
+      # @param [String, Pubid::Identifier] id ID to search for
+      #
+      # @return [Array<Hash>] search results
+      #
+      def search(id = nil, &block)
+        items = search_candidates(id)
+        return items.select(&block) if block
+
+        items.select { |i| match_item(i, id) }
+      end
+
+      #
+      # Save index to storage
+      #
+      # @return [void]
+      #
+      def save
+        @file_io.save(@index || [])
+      end
+
+      #
+      # Remove index file from storage and clear index
+      #
+      # @return [void]
+      #
+      def remove_file
+        @file_io.remove
+        @index = nil
+        @id_lookup = nil
+      end
+
+      #
+      # Remove all index items
+      #
+      # @return [void]
+      #
+      def remove_all
+        @index = []
+        @id_lookup = nil
+        @file_io.sorted = true
+      end
+
+      private
+
+      def id_lookup
+        @id_lookup ||= index.each_with_object({}) do |item, h|
+          h[item[:id].to_s] = item
+        end
+      end
+
+      def search_candidates(id)
+        # index needs to be created to check if sorted
+        idx = index
+        if @file_io.sorted && id && !id.is_a?(String)
+          candidates_by_number(id)
+        else
+          idx
+        end
+      end
+
+      def candidates_by_number(id)
+        target = get_id_number(id)
+        left = bsearch_left(target)
+        return [] unless left
+
+        right = bsearch_right(target)
+        index[left...right]
+      end
+
+      def bsearch_left(target)
+        index.bsearch_index do |item|
+          get_id_number(item[:id]) >= target
+        end
+      end
+
+      def bsearch_right(target)
+        index.bsearch_index do |item|
+          get_id_number(item[:id]) > target
+        end || index.size
+      end
+
+      def match_item(item, id)
+        if item[:id].is_a?(String)
+          item[:id].include?(id.is_a?(String) ? id : id.to_s)
+        elsif id.is_a?(String)
+          item[:id].to_s.include?(id)
+        else
+          item[:id] == id
+        end
+      end
+    end
+  end
+end