relaton 3.0.0.pre.alpha.13 → 3.0.0.pre.alpha.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69f8e17f2c0ce9465b7cc3b648507007c8d0f1efd6d1005370aebcf7f6d10526
-  data.tar.gz: 9d020e7b59a8e08876262d9079df95dfa781954cd8583c32a19ea2c85c660b53
+  metadata.gz: e2c8ca4e0641e619f634a850d3a4001e2077e9cff815329d69e1497c46f323ed
+  data.tar.gz: 47cc2a2852bfa5348229f6e63b7ff08195f2b0b5a558ea1d563a03f768c15ba3
 SHA512:
-  metadata.gz: 1d4121f0ec164ba41a1ff23dfa51dc7c995bf8a90852ad8ca27ac6221fa5547172997561c9e2790996cb9405c89b7662cfdabf88de07bcf3a43011213b9f7de3
-  data.tar.gz: f2bad59aaae815eca687007cbc34d6920c9ca4706e67b50e912bf40579f27f17b4e658fce81923a2899ae74c127e85788fd28518cb12d4233bd4053364ae6ca5
+  metadata.gz: 5ebecc67f9bba5b19b363ac653283701f1ff887a08b6de119adfd7f7847d694899227d058fe362b3a28cd479f319edb7b805ad702ee9b7a9b3c077ae42ce2b15
+  data.tar.gz: e491a0b7e33fc1f38280891933465391225d0710a24b62bb732e5f7414ca037d5ab65f3d83f907323636c1535ecfd2a35d9b74d76bfe65ebf7f3f171d0665adf

data/lib/relaton/3gpp/bibliography.rb

@@ -123,10 +123,15 @@ module Relaton
       # deserialize the rows into identifiers, sort them by `id.root.number`,
       # and let `Type#search` bsearch. Without it the rows stay raw hashes,
       # `FileIO#sorted` stays false, and every lookup scans all 88,464 of them.
+      #
+      # The machine index on the Pages site (relaton#206, the W3C pilot's
+      # contract): a parsed query reads only its own shard. A Pages failure
+      # raises `Relaton::RequestError`; the whole-index zip on the same site
+      # serves only the retry on a miss.
+      #
       def index
         Relaton::Index.find_or_create(
-          "3GPP", url: "#{SOURCE}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
-          pubid_class: ::Pubid::Tgpp::Identifier
+          "3GPP", pages_url: PAGES_URL, pubid_class: ::Pubid::Tgpp::Identifier
         )
       end
 

data/lib/relaton/3gpp.rb

@@ -16,6 +16,10 @@ require_relative "3gpp/bibliography"
 module Relaton
   module ThreeGpp
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by Bibliography's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-3gpp/"
 
     # Returns hash of XML reammar
     # @return [String]

data/lib/relaton/adobe/bibliography.rb

@@ -70,10 +70,13 @@ module Relaton
         end
 
         def index
+          # The machine index on the Pages site (relaton#206): a parsed query
+          # reads only its own shard; a Pages failure raises
+          # `Relaton::RequestError`; the whole-index zip on the same site
+          # serves only the retry on a miss.
           Relaton::Index.find_or_create(
             :adobe,
-            url: "#{ENDPOINT}#{INDEXFILE}.zip",
-            file: "#{INDEXFILE}.yaml",
+            pages_url: PAGES_URL,
             pubid_class: ::Pubid::Adobe::Identifier,
           )
         end

data/lib/relaton/adobe.rb

@@ -13,6 +13,10 @@ require "relaton/version"
 module Relaton
   module Adobe
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-adobe/"
 
     class Error < StandardError; end
 

data/lib/relaton/bib/model/item_shared.rb

@@ -12,7 +12,11 @@ module Relaton
           graphic_work music patent inbook incollection inproceedings journal website
           webresource dataset archival social_media alert message convesation misc
         ]
-        attribute :formattedref, Formattedref
+        # Lenient ingest: some upstream corpus documents carry more than one
+        # <formattedref> in a bibitem; keep the first instead of failing the
+        # whole ingest (lutaml-model#971, the singular_tolerance option in
+        # 0.8.112).
+        attribute :formattedref, Formattedref, singular_tolerance: :first
         attribute :title, Title, collection: true, initialize_empty: true
         attribute :source, Uri, collection: true, initialize_empty: true
         attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true

data/lib/relaton/bib/model/localized_string.rb

@@ -32,6 +32,16 @@ module Relaton
         end
       end
 
+      # Cast at the assignment layer (lutaml-model#972): a bare scalar
+      # assigned to a singular custom-typed attribute (e.g. a relation's
+      # description) is wrapped here, so the serializer's type validation
+      # sees an instance instead of raising IncorrectModelError.
+      def self.cast(value)
+        return value if value.is_a?(self)
+
+        new(content: value.to_s)
+      end
+
       attribute :content, :string
       prepend ContentSanitization
 

data/lib/relaton/bipm/bibliography.rb

@@ -215,8 +215,12 @@ module Relaton::Bipm
       end
 
       def index
+        # The machine index on the Pages site (relaton#206): a parsed query
+        # reads only its own shard; a Pages failure raises
+        # `Relaton::RequestError`; the whole-index zip on the same site
+        # serves only the retry on a miss.
         Relaton::Index.find_or_create(
-          :bipm, url: "#{GH_ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+          :bipm, pages_url: PAGES_URL,
           pubid_class: ::Pubid::Bipm::Identifier
         )
       end

data/lib/relaton/bipm.rb

@@ -23,6 +23,10 @@ module Relaton
     # by `relaton-data-bipm`'s crawler (using the retained `Relaton::Bipm::Id`),
     # but this flavor no longer reads or writes it.
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-bipm/"
 
     # Returns hash of gems versions used to generate the data model.
     # @return [String]

data/lib/relaton/calconnect/hit_collection.rb

@@ -12,9 +12,14 @@ module Relaton::Calconnect
     private
 
     # @return [Relaton::Index::Type]
+    #
+    # The machine index on the Pages site (relaton#206, the W3C pilot's
+    # contract): a parsed query reads only its own shard. A Pages failure
+    # raises `Relaton::RequestError`; the whole-index zip on the same site
+    # serves only the retry on a miss.
+    #
     def index
-      Relaton::Index.find_or_create :CC, url: "#{GHURL}#{INDEXFILE}.zip",
-                                         file: "#{INDEXFILE}.yaml",
+      Relaton::Index.find_or_create :CC, pages_url: PAGES_URL,
                                          pubid_class: ::Pubid::Calconnect::Identifier
     end
 

data/lib/relaton/calconnect.rb

@@ -23,6 +23,10 @@ module Relaton
     # `relaton-data-calconnect`'s crawler derives the legacy `index-v1` from
     # these rows for released consumers, so it is not produced or read here.
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by HitCollection's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-calconnect/"
 
     # Returns hash of XML reammar
     # @return [String]

data/lib/relaton/ccsds/hit_collection.rb

@@ -20,8 +20,12 @@ module Relaton
       # Pubid index (index-v2): `:id` deserializes to a Pubid::Ccsds::Identifier
       # via pubid_class, so search narrows by number with binary search.
       def index
+        # The machine index on the Pages site (relaton#206): a parsed query
+        # reads only its own shard; a Pages failure raises
+        # `Relaton::RequestError`; the whole-index zip on the same site
+        # serves only the retry on a miss.
         @index ||= Relaton::Index.find_or_create(
-          :ccsds, url: "#{GHURL}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml", pubid_class: Pubid::Ccsds::Identifier
+          :ccsds, pages_url: PAGES_URL, pubid_class: Pubid::Ccsds::Identifier
         )
       end
 

data/lib/relaton/ccsds.rb

@@ -19,6 +19,10 @@ module Relaton
     # index-v1 (pubid-v1 hash, for the released gem line) is rebuilt separately
     # by the data repo's build_index_v1.rb.
     INDEXFILE = "index-v2"
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-ccsds/"
 
     class Error < StandardError; end
     # Your code goes here...

data/lib/relaton/cie/scrapper.rb

@@ -15,7 +15,11 @@ module Relaton
           # "... is not a recognized standards identifier". Partial refs
           # (`CIE 001`, `CIE 15`) parse, so nothing valid is lost.
           pubid = to_pubid ref
-          index = Index.find_or_create :cie, url: "#{ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+          # The machine index on the Pages site (relaton#206): a parsed query
+          # reads only its own shard; a Pages failure raises
+          # `Relaton::RequestError`; the whole-index zip on the same site
+          # serves only the retry on a miss.
+          index = Index.find_or_create :cie, pages_url: PAGES_URL,
                                               pubid_class: ::Pubid::Cie::Identifier
           # Pass the parsed pubid (not the raw String) so index-v2 narrows
           # candidates by number via binary search before the block runs; the

data/lib/relaton/cie.rb

@@ -16,6 +16,10 @@ require_relative "cie/scrapper"
 module Relaton
   module Cie
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-cie/"
 
     # Returns hash of XML reammar
     # @return [String]

data/lib/relaton/db/cache.rb

@@ -3,7 +3,6 @@ require "digest"
 require "json"
 require "date"
 require "lutaml/store"
-require "nokogiri"
 require_relative "cache_entry"
 
 module Relaton
@@ -335,12 +334,18 @@ module Relaton
 
       # The wrapped string key of an old-layout document: its own primary
       # docidentifier under its flavor prefix, resolved through the same
-      # `wrapped_key` path a string cache read takes.
+      # `wrapped_key` path a string cache read takes. A value no parser reads
+      # is no document — the legacy cache predates this gem's own writes —
+      # not a crash.
       def old_layout_key(path, value)
         flavor = path.split("/").first or return
-        doc = Nokogiri::XML(value)
-        id = doc.root&.at("./docidentifier[@primary = 'true']") ||
-          doc.root&.at("./docidentifier") or return
+        root = begin
+          Moxml.new.parse(value, readonly: true).root
+        rescue Moxml::ParseError
+          nil
+        end or return
+        id = root.at_xpath("./docidentifier[@primary = 'true']") ||
+          root.at_xpath("./docidentifier") or return
 
         "#{flavor.upcase}(#{id.text.strip})"
       end

data/lib/relaton/db/registry.rb

@@ -13,7 +13,7 @@ module Relaton
         relaton/oasis relaton/doi relaton/jis relaton/xsf
         relaton/ccsds relaton/etsi relaton/isbn relaton/plateau
         relaton/oiml relaton/jcgm relaton/easc relaton/gost relaton/adobe
-        relaton/iala
+        relaton/iala relaton/evs
       ].freeze
 
       include Singleton

data/lib/relaton/easc/bibliography.rb

@@ -68,10 +68,13 @@ module Relaton
         private
 
         def index
+          # The machine index on the Pages site (relaton#206): a parsed query
+          # reads only its own shard; a Pages failure raises
+          # `Relaton::RequestError`; the whole-index zip on the same site
+          # serves only the retry on a miss.
           Relaton::Index.find_or_create(
             :easc,
-            url: "#{ENDPOINT}#{INDEXFILE}.zip",
-            file: "#{INDEXFILE}.yaml",
+            pages_url: PAGES_URL,
             pubid_class: ::Pubid::Easc::Identifier,
           )
         end

data/lib/relaton/easc.rb

@@ -13,6 +13,10 @@ require "pubid"
 module Relaton
   module Easc
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-easc/"
 
     class Error < StandardError; end
 

data/lib/relaton/ecma/bibliography.rb

@@ -11,8 +11,12 @@ module Relaton
       class << self
         # @return [Relaton::Index::Type]
         def index
+          # The machine index on the Pages site (relaton#206): a parsed query
+          # reads only its own shard; a Pages failure raises
+          # `Relaton::RequestError`; the whole-index zip on the same site
+          # serves only the retry on a miss.
           Relaton::Index.find_or_create(
-            :ECMA, url: "#{ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+            :ECMA, pages_url: PAGES_URL,
             pubid_class: ::Pubid::Ecma::Identifier
           )
         end

data/lib/relaton/ecma.rb

@@ -20,6 +20,10 @@ module Relaton
     # `relaton-data-ecma`'s crawler derives the legacy `index-v1` from these
     # rows for released consumers, so it is not produced or read here.
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-ecma/"
 
     # Returns hash of XML reammar
     # @return [String]

data/lib/relaton/etsi/bibliography.rb

@@ -16,7 +16,11 @@ module Relaton
         # A parsed pubid comes from Relaton::Db (relaton#205); it is only read.
         pubid = ref.is_a?(String) ? ::Pubid::Etsi.parse(ref) : ref
 
-        index = Relaton::Index.find_or_create :etsi, url: "#{SOURCE}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+        # The machine index on the Pages site (relaton#206): a parsed query
+        # reads only its own shard; a Pages failure raises
+        # `Relaton::RequestError`; the whole-index zip on the same site
+        # serves only the retry on a miss.
+        index = Relaton::Index.find_or_create :etsi, pages_url: PAGES_URL,
                                                      pubid_class: ::Pubid::Etsi::Identifier
         row = best_match(index, pubid)
         return unless row

data/lib/relaton/etsi.rb

@@ -14,6 +14,10 @@ require_relative "etsi/bibliography"
 module Relaton
   module Etsi
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-etsi/"
 
     # Returns hash of gem versions used to generate data model.
     # @return [String]

data/lib/relaton/evs/bibliography.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Evs
+    # Class methods for search Estonian standards.
+    class Bibliography
+      class << self
+        # @param ref [String] the printed reference
+        # @return [Relaton::Evs::HitCollection]
+        def search(ref, year = nil)
+          HitCollection.new(ref, year).search
+        rescue Net::ReadTimeout, Net::OpenTimeout => e
+          raise Relaton::RequestError, e.message
+        end
+
+        #
+        # Parses a printed EVS reference.
+        #
+        # Deliberately does NOT rescue. An unrecognized query reference must
+        # raise, so that relaton-cli can print `"…" is not a recognized
+        # standards identifier`; returning nil would make "malformed" look
+        # like "not found". Only the DATA side rescues — see `Hit#pubid`.
+        #
+        # @param ref [String] a printed reference, e.g. "EVS-EN 18216:2026"
+        #
+        # @return [Pubid::Evs::Identifier]
+        # @raise [Pubid::Errors::ParseError]
+        #
+        def parse(ref)
+          ::Pubid::Evs.parse ref
+        end
+
+        #
+        # @param ref [String, Pubid::Evs::Identifier] the EVS standard
+        #   reference to look up, or its parse from Relaton::Db (relaton#205)
+        # @param year [String] the year the standard was published (optional)
+        # @param opts [Hash] options
+        # @option opts [Boolean] :keep_year don't update reference
+        #
+        # @return [Relaton::Evs::ItemData, nil]
+        #
+        def get(ref, year = nil, opts = {})
+          # An empty string is no reference, not a malformed one, so it stays a
+          # plain nil rather than a parse failure. Mirrors the guard
+          # HitCollection#search already carries.
+          return if ref.nil? || (ref.is_a?(String) && ref.strip.empty?)
+
+          # `Relaton::Db` hands `get` the parse it routed and keyed the cache
+          # with; it must reach `bib_get` unmutated.
+          query = ref.is_a?(String) ? parse(ref) : ref
+          year ||= publication_year query
+
+          bib_get ref.to_s, year, opts, query
+        end
+
+        private
+
+        #
+        # `#root` is the accessor that answers for every form: a national
+        # adoption keeps the origin standard's number and year behind its
+        # `base` chain, and a direct adoption or plain Estonian standard is
+        # its own root.
+        #
+        # @param id [Pubid::Evs::Identifier]
+        #
+        # @return [String, nil]
+        #
+        def publication_year(id)
+          id.root.year&.to_s
+        end
+
+        # A component is absent from a reference when excluding it changes
+        # nothing.
+        #
+        # @return [Boolean]
+        def absent?(id, *keys)
+          id.exclude(*keys) == id
+        end
+
+        def fetch_ref_err(ref, year, missed_years)
+          unless missed_years.empty?
+            Util.info "There was no match for `#{year}`, though there " \
+                      "were matches found for `#{missed_years.join('`, `')}`.",
+                      key: ref
+          end
+          nil
+        end
+
+        #
+        # Selects the catalogue hits that denote the same document as the
+        # query. The selection is pubid's subset match `query === hit.pubid`:
+        # a year the query omits matches any value, and the class must be
+        # identical, so a base reference never answers with its own
+        # amendment's record.
+        #
+        # @param ref [String] the printed reference, as the catalogue URL
+        #   wants it
+        # @param query [Pubid::Evs::Identifier]
+        #
+        # @return [Relaton::Evs::HitCollection]
+        #
+        def search_filter(ref, query)
+          search(ref).select! { |hit| hit.pubid && query === hit.pubid }
+        end
+
+        # Fetch the matching hit, keeping the year filter for error reporting.
+        def isobib_results_filter(result, year)
+          missed_years = []
+          result.each do |r|
+            pyear = r.pubid && publication_year(r.pubid)
+            if !year || year == pyear
+              ret = r.item
+              return { ret: ret } if ret
+            end
+
+            missed_years << pyear
+          end
+          { years: missed_years }
+        end
+
+        def bib_get(ref, year, opts, query)
+          key = year && absent?(query, :year) ? "#{ref}:#{year}" : ref
+          Util.info "Fetching from www.evs.ee ...", key: key
+          result = search_filter(ref, query)
+          ret = isobib_results_filter(result, year)
+          if ret[:ret]
+            bib = year || opts[:keep_year] ? ret[:ret] : ret[:ret].to_most_recent_reference
+            Util.info "Found: `#{bib.docidentifier.first&.content}`", key: key
+            bib
+          else
+            Util.info "Not found.", key: key
+            fetch_ref_err(ref, year, ret[:years])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/relaton/evs/hit.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Evs
+    # Hit.
+    class Hit < Relaton::Core::Hit
+      attr_writer :item
+
+      # Parse page.
+      # @return [Relaton::Evs::ItemData]
+      def item
+        @fetch ||= Scraper.parse_page self
+      end
+
+      # The hit's code parsed with pubid, memoized because the filter, the
+      # sort and the scraper all read it. `nil` when pubid cannot parse the
+      # code: the catalogue also hosts documents outside the EVS grammar
+      # (a bare `EN` or `ISO` record linked as a base document), and one such
+      # row must not abort the search. (A malformed *query* raises; see
+      # `Bibliography.parse`.)
+      #
+      # @return [Pubid::Evs::Identifier, nil]
+      def pubid
+        return @pubid if defined?(@pubid)
+
+        @pubid = Bibliography.parse hit[:code]
+      rescue StandardError
+        @pubid = nil
+      end
+    end
+  end
+end

data/lib/relaton/evs/hit_collection.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Evs
+    # Page of hit collection.
+    class HitCollection < Relaton::Core::HitCollection
+      DOMAIN = "https://www.evs.ee"
+
+      def search
+        @array = hits
+        self
+      end
+
+      def agent
+        @agent ||= Mechanize.new.tap { |a| a.user_agent_alias = "Mac Safari" }
+      end
+
+      def select!(&block)
+        @array.select!(&block)
+        self
+      end
+
+      private
+
+      #
+      # The catalogue URL is the printed reference slugified
+      # (`EVS-EN 18216:2026` -> /en/evs-en-18216-2026), so the search is a
+      # direct probe: the product page exists iff the reference denotes a
+      # published document. A miss redirects to /page-not-found, which is no
+      # hit. A reference carrying the `et` language code probes the Estonian
+      # locale, which serves the Estonian-language product.
+      #
+      # @return [Array<Relaton::Evs::Hit>]
+      #
+      def hits
+        page = agent.get url
+        return [] if page.uri.to_s.include? "page-not-found"
+
+        [Hit.new({ code: code, url: page.uri.to_s }, self)]
+      rescue Mechanize::ResponseCodeError
+        []
+      end
+
+      def url
+        "#{DOMAIN}/#{locale}/#{slug}"
+      end
+
+      def locale
+        query_pubid&.language == "et" ? "et" : "en"
+      end
+
+      # The hit's code is the reference as queried, minus the "(all parts)"
+      # marker: `Hit#pubid` parses it back, and the filter `query === hit.pubid`
+      # needs the language code ON the hit for a language-carrying query to
+      # match its own record. The slug derivation strips the language code
+      # separately — the catalogue URL does not carry it.
+      def code
+        ref.to_s.sub(/\s+\(all parts\)\z/, "")
+      end
+
+      def slug
+        code.sub(/\s+(et-en|et|en)\z/, "")
+            .downcase.gsub(/[^a-z0-9]+/, "-").gsub(/\A-+|-+\z/, "")
+      end
+
+      def query_pubid
+        Bibliography.parse code
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/relaton/evs/item_data.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Evs
+    # Item data class.
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/evs/model/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Evs
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/evs/model/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Evs
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end