relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/iho/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Iho
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/iho/processor.rb

@@ -0,0 +1,54 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Iho
+    class Processor < Core::Processor
+      attr_reader :idtype
+
+      def initialize
+        @short = :relaton_iho
+        @prefix = "IHO"
+        @defaultprefix = %r{^IHO\s}
+        @idtype = "IHO"
+      end
+
+      # @param code [String]
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [Relaton::Iho::ItemData, nil]
+      def get(code, date, opts)
+        require_relative "../iho"
+        Bibliography.get(code, date, opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Iho::ItemData]
+      def from_xml(xml)
+        require_relative "../iho"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Iho::ItemData]
+      def from_yaml(yaml)
+        require_relative "../iho"
+        Item.from_yaml(yaml)
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../iho"
+        @grammar_hash ||= ::Relaton::Iho.grammar_hash
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../iho"
+        Relaton::Index.find_or_create(:iho, url: true, file: "#{INDEXFILE}.yaml").remove_file
+      end
+    end
+  end
+end

data/lib/relaton/iho/relation.rb

@@ -0,0 +1,9 @@
+require_relative "item_base"
+
+module Relaton
+  module Iho
+    class Relation < Bib::Relation
+      attribute :bibitem, ItemBase
+    end
+  end
+end

data/lib/relaton/iho/structured_identifier.rb

@@ -0,0 +1,20 @@
+module Relaton
+  module Iho
+    class StructuredIdentifier < Lutaml::Model::Serializable
+      attribute :docnumber, :string
+      attribute :part, :string
+      attribute :annexid, :string
+      attribute :appendixid, :string
+      attribute :supplementid, :string
+
+      xml do
+        root "structuredidentifier"
+        map_element "docnumber", to: :docnumber
+        map_element "part", to: :part
+        map_element "annexid", to: :annexid
+        map_element "appendixid", to: :appendixid
+        map_element "supplementid", to: :supplementid
+      end
+    end
+  end
+end

data/lib/relaton/iho/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Iho
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-iho".freeze
+    end
+  end
+end

data/lib/relaton/iho.rb

@@ -0,0 +1,27 @@
+require "relaton/bib"
+require "relaton/index"
+require "pubid"
+require_relative "version"
+require_relative "iho/util"
+require_relative "iho/docidentifier"
+require_relative "iho/item"
+require_relative "iho/bibitem"
+require_relative "iho/bibdata"
+require_relative "iho/bibliography"
+
+module Relaton
+  module Iho
+    INDEXFILE = "index-v3".freeze
+
+    class Error < StandardError; end
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/index/config.rb

@@ -0,0 +1,52 @@
+module Relaton
+  module Index
+    #
+    # Configuration class for Relaton::Index
+    #
+    class Config
+      attr_reader :storage, :storage_dir, :filename
+
+      #
+      # Set default values
+      #
+      def initialize
+        @storage = FileStorage
+        @storage_dir = Dir.home
+        @filename = "index.yaml"
+      end
+
+      #
+      # Set storage
+      #
+      # @param [#ctime, #read, #write] storage storage object
+      #
+      # @return [void]
+      #
+      def storage=(storage)
+        @storage = storage
+      end
+
+      #
+      # Set storage directory
+      #
+      # @param [String] dir storage directory
+      #
+      # @return [void]
+      #
+      def storage_dir=(dir)
+        @storage_dir = dir
+      end
+
+      #
+      # Set filename
+      #
+      # @param [String] filename filename
+      #
+      # @return [void]
+      #
+      def filename=(filename)
+        @filename = filename
+      end
+    end
+  end
+end

data/lib/relaton/index/file_io.rb

@@ -0,0 +1,305 @@
+module Relaton
+  module Index
+    #
+    # File IO class is used to read and write index files.
+    # In searh mode url is used to fetch index from external repository and save it to storage.
+    # In index mode url should be nil.
+    #
+    class FileIO
+      # Raised internally when a deserialized id cannot be parsed or is not
+      # understood by the pubid class; `#load_index` rescues it to trigger the
+      # wrong-structure handling (re-download, or stop and log).
+      class InvalidIndexError < StandardError; end
+
+      attr_reader :url, :pubid_class
+      attr_accessor :sorted
+
+      @@file_locks = {}
+      @@file_locks_mutex = Mutex.new
+
+      #
+      # Initialize FileIO
+      #
+      # @param [String] dir falvor specific local directory in ~/.relaton to store index
+      # @param [String, Boolean, nil] url
+      #   if String then the URL is used to fetch an index from a Git repository
+      #     and save it to the storage (if not exists, or older than 24 hours)
+      #   if true then the index is read from the storage (used to remove index file)
+      #   if nil then the fiename is used to read and write file (used to create indes in GH actions)
+      # @param [Pubid::Identifier] pubid class for deserialization
+      #
+      # `id_keys` is accepted for backward compatibility but no longer used: the
+      # index format is now validated by round-tripping a sample of ids through
+      # the pubid class (see #check_serialization), which understands the pubid
+      # v2 (lutaml) `_type` serialization that the old key-allowlist could not.
+      def initialize(dir, url, filename, _id_keys = nil, pubid_class = nil)
+        @dir = dir
+        @url = url
+        @filename = filename
+        @pubid_class = pubid_class
+        @sorted = false
+      end
+
+      #
+      # If url is String, check if index file exists and is not older than 24
+      #   hours. If not, fetch index from external repository and save it to
+      #   storage.
+      # If url is true, read index from path to local file.
+      # If url is nil, read index from filename.
+      #
+      # @return [Array<Hash>] index
+      #
+      def read
+        case url
+        when String
+          with_file_lock do
+            check_file || fetch_and_save
+          end
+        else
+          read_file || []
+        end
+      end
+
+      def file
+        @file ||= url ? path_to_local_file : @filename
+      end
+
+      #
+      # Create path to local file
+      #
+      # @return [<Type>] <description>
+      #
+      def path_to_local_file
+        File.join(Index.config.storage_dir, ".relaton", @dir, @filename)
+      end
+
+      #
+      # Check if index file exists and is not older than 24 hours
+      #
+      # @return [Array<Hash>, nil] index or nil
+      #
+      def check_file
+        ctime = Index.config.storage.ctime(file)
+        return unless ctime && ctime > Time.now - 86400
+
+        read_file
+      end
+
+      #
+      # Check if index has correct format
+      #
+      # @param [Array<Hash>] index index to check
+      #
+      # @return [Boolean] <description>
+      #
+      # Structural check only. Per-id serialization is validated during
+      # deserialization (see #deserialize_id), which reuses the `from_hash` the
+      # index load performs anyway, so every row is checked at no extra parse
+      # cost.
+      def check_format(index)
+        check_basic_format(index)
+      end
+
+      def check_basic_format(index)
+        return false unless index.is_a? Array
+
+        keys = %i[file id]
+        index.all? { |item| item.respond_to?(:keys) && item.keys.sort == keys }
+      end
+
+      # An id is supported when `from_hash` either resolves it to a concrete
+      # type (a subclass — the polymorphic `_type` matched) or round-trips
+      # losslessly through `to_hash`. The subclass clause covers valid entries
+      # pubid cannot fully rebuild on re-serialize (e.g. ISO directives drop a
+      # redundant subgroup number); the round-trip clause covers pubid classes
+      # without a subclass hierarchy. A wrong-format/garbled id satisfies
+      # neither: it falls back to the bare base class and fails to round-trip.
+      def id_supported?(obj, raw)
+        # A concrete subtype means pubid recognized the `_type`; accept without
+        # round-tripping. This both skips the false positive for valid-but-lossy
+        # types (e.g. ISO directives) and avoids the costly hash compare for the
+        # ~all rows that resolve to a subtype (it would otherwise add ~33%).
+        return true unless obj.instance_of?(@pubid_class)
+
+        normalize(obj.to_hash) == normalize(raw)
+      rescue StandardError
+        false
+      end
+
+      # Stringify hash keys and scalar values so the comparison ignores YAML
+      # scalar typing (e.g. 1 vs "1") and string/symbol key differences, while
+      # still detecting dropped/added keys or genuinely changed values.
+      def normalize(value)
+        case value
+        when Hash then value.to_h { |k, v| [k.to_s, normalize(v)] }
+        when Array then value.map { |v| normalize(v) }
+        when nil then nil
+        else value.to_s
+        end
+      end
+
+      #
+      # Read index from storage
+      #
+      # @return [Array<Hash>] index
+      #
+      def read_file
+        yaml = Index.config.storage.read(file)
+        return unless yaml
+
+        load_index(yaml) || []
+      end
+
+      # Deserialize and sort by the same narrowing key Type#search bsearches on
+      # — the base document's number, `id.root.number.to_s` (see
+      # Type#candidates_by_number) — so binary search always has a consistent
+      # total order. The published index is only approximately sorted (generated
+      # under pubid 1.x base semantics); merely detecting sortedness left bsearch
+      # disabled and every search a full O(n) scan. Sorting here is one-time per
+      # load.
+      def deserialize_pubid(index)
+        return index unless @pubid_class
+
+        deserialized = index.map do |r|
+          { id: deserialize_id(r[:id]), file: r[:file] }
+        end
+        warn_unless_sorted(deserialized)
+        deserialized.sort_by! { |r| r[:id].root.number.to_s }
+        @sorted = true
+        deserialized
+      end
+
+      # Deserialize one id and verify pubid understands it. Reuses the
+      # `from_hash` deserialization the load performs anyway, so validating every
+      # row costs only the `to_hash`/compare for ids that need the round-trip
+      # clause. Raises InvalidIndexError when an id cannot be parsed or is
+      # unsupported, so `#load_index` rejects (and re-downloads) the whole index.
+      def deserialize_id(raw)
+        obj = @pubid_class.from_hash(raw)
+      rescue StandardError => e
+        raise InvalidIndexError, "cannot parse id #{raw.inspect}: #{e.message}"
+      else
+        return obj if id_supported?(obj, raw)
+
+        raise InvalidIndexError, "unsupported id #{raw.inspect}"
+      end
+
+      # Log when the loaded index is not already in narrowing-key order, so the
+      # in-memory sort above (and the underlying not-sorted index file) is
+      # visible. Stops at the first out-of-order pair.
+      def warn_unless_sorted(index)
+        prev = nil
+        index.each do |r|
+          num = r[:id].root.number.to_s
+          if prev && prev > num
+            Util.warn "Index file `#{file}` is not sorted by id number; " \
+                      "sorting #{index.size} entries in memory.", progname
+            return
+          end
+          prev = num
+        end
+      end
+
+      def warn_local_index_error(reason)
+        Util.info "#{reason} file `#{file}`", progname
+        if url.is_a? String
+          Util.info "Considering `#{file}` file corrupt, re-downloading from `#{url}`", progname
+        else
+          Util.info "Considering `#{file}` file corrupt, removing it.", progname
+          remove
+        end
+      end
+
+      def progname
+        @progname ||= "relaton-#{@dir}"
+      end
+
+      def load_index(yaml, save = false)
+        index = YAML.safe_load(yaml, permitted_classes: [Symbol])
+        save index if save
+        return deserialize_pubid(index) if check_format(index)
+
+        report_invalid_index(save, "Wrong structure of")
+      rescue Psych::SyntaxError
+        report_invalid_index(save, "YAML parsing error when reading")
+      rescue InvalidIndexError
+        report_invalid_index(save, "Wrong structure of")
+      end
+
+      def report_invalid_index(save, reason)
+        if save
+          warn_remote_index_error reason
+        else
+          warn_local_index_error reason
+        end
+      end
+
+      #
+      # Fetch index from external repository and save it to storage
+      #
+      # @return [Array<Hash>] index
+      #
+      def fetch_and_save
+        uri = URI.parse(url)
+        body = Net::HTTP.get(uri)
+        yaml = nil
+        Zip::File.open_buffer(body) do |zip|
+          entry = zip.entries.first
+          yaml = entry.get_input_stream.read
+        end
+        Util.info "Downloaded index from `#{url}`", progname
+        load_index(yaml, true)
+      end
+
+      def warn_remote_index_error(reason)
+        Util.info "#{reason} newly downloaded file `#{file}` at `#{url}`, " \
+             "the remote index seems to be invalid. Please report this " \
+             "issue at https://github.com/relaton/relaton-cli.", progname
+      end
+
+      #
+      # Save index to storage
+      #
+      # @param [Array<Hash>] index index to save
+      #
+      # @return [void]
+      #
+      def save(index)
+        yaml = sort_structured_index(index).map do |item|
+          item.transform_values do |value|
+            @pubid_class && value.is_a?(@pubid_class) ? value.to_hash : value
+          end
+        end.to_yaml
+        Index.config.storage.write file, yaml
+      end
+
+      def sort_structured_index(index)
+        if @pubid_class && index.first&.dig(:id).is_a?(@pubid_class)
+          index.sort_by { |item| item[:id].root.number.to_s }
+        else
+          index
+        end
+      end
+
+      #
+      # Remove index file from storage
+      #
+      # @return [Array]
+      #
+      def remove
+        Index.config.storage.remove file
+        []
+      end
+
+      private
+
+      def with_file_lock(&)
+        @@file_locks_mutex.synchronize do
+          @@file_locks[file] ||= Mutex.new
+        end
+
+        @@file_locks[file].synchronize(&)
+      end
+    end
+  end
+end

data/lib/relaton/index/file_storage.rb

@@ -0,0 +1,66 @@
+module Relaton
+  module Index
+    #
+    # File storage module contains methods to read and write files
+    #
+    module FileStorage
+      #
+      # Return file creation time
+      #
+      # @param [String] file file path
+      #
+      # @return [Time, nil] file creation time or nil if file does not exist
+      #
+      def ctime(file)
+        File.exist?(file) && File.ctime(file)
+      end
+
+      #
+      # Read file
+      #
+      # @param [String] file file path
+      #
+      # @return [String, nil] file content or nil if file does not exist
+      #
+      def read(file)
+        return unless File.exist?(file)
+
+        File.read file, encoding: "UTF-8"
+      end
+
+      #
+      # Write file
+      #
+      # @param [String] file file path
+      # @param [String] data content to write
+      #
+      # @return [void]
+      #
+      def write(file, data)
+        dir = File.dirname file
+        FileUtils.mkdir_p dir
+        # Write the bytes verbatim. `data` is often a raw Net::HTTP body, an
+        # ASCII-8BIT string; passing `encoding: "UTF-8"` would *transcode* it and
+        # raise Encoding::UndefinedConversionError on any non-ASCII byte (e.g. a
+        # UTF-8 `\xC3` in "électrotechnique"). The bytes are already valid UTF-8
+        # and `read` decodes them as UTF-8, so a binary write round-trips.
+        File.binwrite file, data
+      end
+
+      #
+      # Remove file
+      #
+      # @param [String] file file path
+      #
+      # @return [void]
+      #
+      def remove(file)
+        return unless File.exist? file
+
+        File.delete file
+      end
+
+      extend self
+    end
+  end
+end

data/lib/relaton/index/pool.rb

@@ -0,0 +1,46 @@
+module Relaton
+  module Index
+    #
+    # Pool of indexes
+    #
+    class Pool
+      def initialize
+        @pool = {}
+      end
+
+      #
+      # Return index by type, create if not exists
+      #
+      # @param [String] type <description>
+      # @param [String, nil] url external URL to index, used to fetch index for searching files
+      # @param [String, nil] file output file name
+      # @param [Array<Symbol>, nil] id_keys keys to check if index is correct
+      # @param [String, nil] pages_url base URL of the Pages site that serves
+      #   the machine index (manifest + shards), see Relaton::Index::ShardSource
+      #
+      # @return [Relaton::Index::Type] typed index
+      #
+      def type(type, **args)
+        if @pool[type.upcase.to_sym]&.actual?(**args)
+          @pool[type.upcase.to_sym]
+        else
+          @pool[type.upcase.to_sym] = Type.new(
+            type, args[:url], args[:file], args[:id_keys], args[:pubid_class],
+            pages_url: args[:pages_url]
+          )
+        end
+      end
+
+      #
+      # Remove index by type from pool
+      #
+      # @param [String] type index type
+      #
+      # @return [void]
+      #
+      def remove(type)
+        @pool.delete type.upcase.to_sym
+      end
+    end
+  end
+end