relaton 3.0.0.pre.alpha.13 → 3.0.0.pre.alpha.15

data/lib/relaton/iso.rb

@@ -19,6 +19,10 @@ require_relative "iso/bibliography"
 module Relaton
   module Iso
     INDEXFILE = "index-v2"
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by HitCollection's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-iso/"
 
     def self.grammar_hash
       # gem_path = File.expand_path "..", __dir__

data/lib/relaton/itu/hit_collection.rb

@@ -144,9 +144,14 @@ module Relaton
         [year.to_i, month.to_i]
       end
 
+      #
+      # The machine index on the Pages site (relaton#206, the W3C pilot's
+      # contract): a parsed query reads only its own shard. A Pages failure
+      # raises `Relaton::RequestError`; the whole-index zip on the same site
+      # serves only the retry on a miss.
+      #
       def index
-        Relaton::Index.find_or_create :itu, url: "#{GH_ITU}#{INDEXFILE}.zip",
-                                            file: "#{INDEXFILE}.yaml",
+        Relaton::Index.find_or_create :itu, pages_url: PAGES_URL,
                                             pubid_class: ::Pubid::Itu::Identifier
       end
 

data/lib/relaton/itu/model/ext.rb

@@ -8,6 +8,14 @@ require_relative "structured_identifier"
 module Relaton
   module Itu
     class Ext < Bib::Ext
+      # The `<flavor>` element is required by the shared ext grammar, and an
+      # ITU record is always flavor "itu" — including a legacy record
+      # deserialized before the producers stamped the attribute, which
+      # carries nil. The derived attribute keeps the element in the
+      # round-tripped XML (a `default:` would be suppressed by the
+      # serializer, omitting the element for every record).
+      attribute :flavor, :string, method: :itu_flavor
+
       attribute :doctype, Doctype
       attribute :structuredidentifier, StructuredIdentifier
       attribute :question, Question, collection: true
@@ -42,6 +50,8 @@ module Relaton
       end
 
       def get_schema_version = Relaton.schema_versions["relaton-model-itu"]
+
+      def itu_flavor = "itu"
     end
   end
 end

data/lib/relaton/itu.rb

@@ -17,6 +17,10 @@ require_relative "itu/bibliography"
 module Relaton
   module Itu
     INDEXFILE = "index-v2"
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by HitCollection's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-itu/"
 
     class Error < StandardError; end
 

data/lib/relaton/jcgm/bibliography.rb

@@ -66,10 +66,13 @@ module Relaton
         end
 
         def index
+          # The machine index on the Pages site (relaton#206): a parsed query
+          # reads only its own shard; a Pages failure raises
+          # `Relaton::RequestError`; the whole-index zip on the same site
+          # serves only the retry on a miss.
           Relaton::Index.find_or_create(
             :jcgm,
-            url: "#{ENDPOINT}#{INDEXFILE}.zip",
-            file: "#{INDEXFILE}.yaml",
+            pages_url: PAGES_URL,
             pubid_class: ::Pubid::Jcgm::Identifier,
           )
         end

data/lib/relaton/jcgm.rb

@@ -12,6 +12,10 @@ require_relative "jcgm/bibliography"
 module Relaton
   module Jcgm
     INDEXFILE = "index-v1".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-jcgm/"
 
     class Error < StandardError; end
 

data/lib/relaton/nist/hit_collection.rb

@@ -60,9 +60,16 @@ module Relaton
       def search_filter # rubocop:disable Metrics/MethodLength
         refid = ::Pubid::Nist::Identifier.parse(@reference)
         parts = exclude_parts refid
+        # A stage-carrying reference matches stage-less rows: the published
+        # index has no stage-typed rows for these documents, and the
+        # from_ga search already fell back to the base document on the
+        # stage-narrowed miss (relaton#146) — so exclude the stage from
+        # both sides of the equality, not only from the hits.
+        parts << :stage if refid.stage
+        compare_refid = refid.stage ? refid.exclude(:stage) : refid
         arr = @array.select do |item|
           pubid = ::Pubid::Nist::Identifier.parse(item.hit[:code])
-          pubid.exclude(*parts) == refid
+          pubid.exclude(*parts) == compare_refid
         rescue StandardError
           item.hit[:code] == ref
         end
@@ -250,6 +257,18 @@ module Relaton
         rows = index.search(r) { |row| row[:id].to_s.include?(needle) }
           .sort_by { |row| row[:id].to_s }
 
+        if rows.empty? && r.is_a?(::Pubid::Nist::Identifier) && r.stage
+          # A stage-carrying reference (the "fpd" of NIST FIPS 197) has no
+          # stage-typed row of its own in the published index — the data repo
+          # indexes the document family by number. Retry with the stage
+          # excluded so the base document answers, as the released
+          # 3.0.0.pre.alpha.1 line did (relaton#146).
+          base = r.exclude(:stage)
+          needle = base.to_s
+          rows = index.search(base) { |row| row[:id].to_s.include?(needle) }
+            .sort_by { |row| row[:id].to_s }
+        end
+
         rows.map do |row|
           # index-v2 stores Pubid objects; the rest of the pipeline (search_filter,
           # Hit, Scraper) works on string codes, so stringify at the boundary.

data/lib/relaton/oasis/bibliography.rb

@@ -50,9 +50,12 @@ module Relaton
         # @return [Relaton::Index::Type]
         #
         def index
+          # The machine index on the Pages site (relaton#206): a parsed query
+          # reads only its own shard; a Pages failure raises
+          # `Relaton::RequestError`; the whole-index zip on the same site
+          # serves only the retry on a miss.
           Relaton::Index.find_or_create(
-            :oasis, url: "#{ENDPOINT}#{INDEXFILE}.zip",
-            file: "#{INDEXFILE}.yaml",
+            :oasis, pages_url: PAGES_URL,
             pubid_class: ::Pubid::Oasis::Identifier
           )
         end

data/lib/relaton/oasis.rb

@@ -25,6 +25,10 @@ module Relaton
     # crawler derives it from the v2 rows for relaton v2 consumers (the
     # IANA/BIPM/W3C/ECMA shape).
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-oasis/"
 
     class Error < StandardError; end
     # Your code goes here...

data/lib/relaton/ogc/hit_collection.rb

@@ -26,10 +26,15 @@ module Relaton
       end
 
       # @return [Relaton::Index::Type]
+      #
+      # The machine index on the Pages site (relaton#206, the W3C pilot's
+      # contract): a parsed query reads only its own shard. A Pages failure
+      # raises `Relaton::RequestError`; the whole-index zip on the same site
+      # serves only the retry on a miss.
+      #
       def index
         @index ||= Relaton::Index.find_or_create(
-          :ogc, url: "#{ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
-          pubid_class: ::Pubid::Ogc::Identifier
+          :ogc, pages_url: PAGES_URL, pubid_class: ::Pubid::Ogc::Identifier
         )
       end
 

data/lib/relaton/ogc.rb

@@ -16,6 +16,10 @@ require_relative "ogc/bibliography"
 module Relaton
   module Ogc
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by HitCollection's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-ogc/"
     class Error < StandardError; end
 
     # Returns hash of XML reammar

data/lib/relaton/oiml/bibliography.rb

@@ -70,10 +70,13 @@ module Relaton
         private
 
         def index
+          # The machine index on the Pages site (relaton#206): a parsed query
+          # reads only its own shard; a Pages failure raises
+          # `Relaton::RequestError`; the whole-index zip on the same site
+          # serves only the retry on a miss.
           Relaton::Index.find_or_create(
             :oiml,
-            url: "#{ENDPOINT}#{INDEXFILE}.zip",
-            file: "#{INDEXFILE}.yaml",
+            pages_url: PAGES_URL,
             pubid_class: ::Pubid::Oiml::Identifier,
           )
         end

data/lib/relaton/oiml.rb

@@ -12,6 +12,10 @@ require_relative "oiml/bibliography"
 module Relaton
   module Oiml
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-oiml/"
 
     class Error < StandardError; end
 

data/lib/relaton/omg/bibliography.rb

@@ -6,10 +6,48 @@ module Relaton
     module Bibliography
       extend self
 
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-omg/main/".freeze
+
       # @param ref [String, Pubid::Omg::Identifier] the OMG standard reference
-      # @return [Relaton::Omg::Item]
+      # @return [Relaton::Omg::Item, nil] nil when neither the machine index
+      #   nor the live scrape answers
       def search(ref)
-        Scraper.scrape_page ref
+        pubid = ref.is_a?(String) ? ::Pubid::Omg::Identifier.parse(ref) : ref
+        Util.info "Fetching from Relaton repository ...", key: pubid.to_s
+        from_index(pubid) || begin
+          # The crawl does not cover every spec (beta-version pages carry no
+          # download link), so a machine-index miss falls back to the live
+          # scrape. Index transport failures still raise — they are not
+          # misses.
+          Util.info "Not in the machine index; scraping www.omg.org ...", key: pubid.to_s
+          Scraper.scrape_page ref
+        end
+      end
+
+      # The machine index on the Pages site (relaton#206): the consumer reads
+      # the published index-v2 (whole-index mode — the manifest publishes no
+      # shards) and fetches the document from the data repo.
+      #
+      # @param pubid [Pubid::Omg::Identifier]
+      # @return [Relaton::Omg::Item, nil]
+      def from_index(pubid)
+        row = index.search(pubid).min_by { |r| r[:id].version.to_s }
+        return unless row
+
+        uri = URI("#{ENDPOINT}#{row[:file]}")
+        resp = Net::HTTP.get_response uri
+        unless resp.code == "200"
+          raise Relaton::RequestError, "Could not access #{uri}: HTTP #{resp.code}"
+        end
+
+        Relaton::Omg::Item.from_yaml resp.body
+      end
+
+      # @return [Relaton::Index::Type]
+      def index
+        Relaton::Index.find_or_create(
+          :omg, pages_url: PAGES_URL, pubid_class: ::Pubid::Omg::Identifier
+        )
       end
 
       # @param ref [String, Pubid::Omg::Identifier] the OMG standard reference

data/lib/relaton/omg/scraper.rb

@@ -38,7 +38,16 @@ module Relaton
         @url += @version.gsub(" ", "/") if @version
         agent = Mechanize.new
         agent.open_timeout = 10
-        @doc = agent.get(@url)
+        begin
+          @doc = agent.get(@url)
+        rescue Mechanize::ResponseCodeError => e
+          raise if e.response_code != "404" || !@part
+
+          # A glued beta label parses as the part ("1.5 Beta1" → version
+          # 1.5, part Beta1) but on omg.org it lives in the version URL
+          # segment — retry with the part appended.
+          @doc = agent.get("#{@url}/#{@part}")
+        end
       rescue Mechanize::ResponseCodeError => e
         return if e.response_code == "404"
 
@@ -155,7 +164,10 @@ module Relaton
             acronym = row.at_xpath("td[3]/a")[:href].split("/")[4]
             id = render_id(acronym, ver)
             docid = Docidentifier.new(content: id, type: "OMG")
-            bibitem = Bib::ItemBase.new(formattedref: Bib::Formattedref.new(content: id), docidentifier: [docid])
+            # Bib::Relation's bibitem is typed ItemData on this model path —
+            # a plain ItemBase fails lutaml's type validation at serialization
+            # (the crawl's "attribute 'bibitem' value is a 'ItemBase'" skips).
+            bibitem = Bib::ItemData.new(formattedref: Bib::Formattedref.new(content: id), docidentifier: [docid])
             mem << Bib::Relation.new(type: "obsoletes", bibitem: bibitem)
           end
           mem

data/lib/relaton/omg.rb

@@ -13,6 +13,17 @@ require_relative "omg/bibliography"
 
 module Relaton
   module Omg
+    # The flavor's published index on relaton-data-omg (relaton#206): the
+    # machine index the Pages site serves, built from the crawled
+    # data/*.yaml with pubid_class ::Pubid::Omg::Identifier. relaton-data-omg
+    # supersedes the flavor's earlier "no index, no data repository" shape —
+    # the crawler and the Pages site are live.
+    INDEXFILE = "index-v2"
+    # The machine index on the data repo's Pages site (relaton#206) — the
+    # manifest + whole-index zip the consumer reads; the crawl does not cover
+    # every spec, so Bibliography falls back to the live scrape on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-omg/"
+
     # Returns hash of XML reammar
     # @return [String]
     def self.grammar_hash

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "3.0.0.pre.alpha.13".freeze
+  VERSION = "3.0.0.pre.alpha.15".freeze
 end

data/lib/relaton/xsf/hit_collection.rb

@@ -31,10 +31,13 @@ module Relaton
       end
 
       def index
+        # The machine index on the Pages site (relaton#206): a parsed query
+        # reads only its own shard; a Pages failure raises
+        # `Relaton::RequestError`; the whole-index zip on the same site
+        # serves only the retry on a miss.
         @index ||= Relaton::Index.find_or_create(
           :xsf,
-          url: "#{GHDATA_URL}#{INDEXFILE}.zip",
-          file: "#{INDEXFILE}.yaml",
+          pages_url: PAGES_URL,
           pubid_class: ::Pubid::Xsf::Identifier,
         )
       end

data/lib/relaton/xsf.rb

@@ -20,6 +20,10 @@ require_relative "xsf/bibliography"
 module Relaton
   module Xsf
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-xsf/"
 
     class Error < StandardError; end
 

data/lib/relaton.rb

@@ -33,6 +33,7 @@ module Relaton
   autoload :Ecma, "relaton/ecma"
   autoload :Etsi, "relaton/etsi"
   autoload :Easc, "relaton/easc"
+  autoload :Evs, "relaton/evs"
   autoload :Gost, "relaton/gost"
   autoload :Iana, "relaton/iana"
   autoload :Iala, "relaton/iala"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre.alpha.13
+  version: 3.0.0.pre.alpha.15
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-09 00:00:00.000000000 Z
+date: 2026-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -98,14 +98,14 @@ dependencies:
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.7'
 - !ruby/object:Gem::Dependency
@@ -442,14 +442,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.pre.alpha.21
+        version: 2.0.0.pre.alpha.28
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.pre.alpha.21
+        version: 2.0.0.pre.alpha.28
 - !ruby/object:Gem::Dependency
   name: yeptris
   requirement: !ruby/object:Gem::Requirement
@@ -809,6 +809,20 @@ files:
 - lib/relaton/etsi/pubid.rb
 - lib/relaton/etsi/status.rb
 - lib/relaton/etsi/util.rb
+- lib/relaton/evs.rb
+- lib/relaton/evs/bibliography.rb
+- lib/relaton/evs/hit.rb
+- lib/relaton/evs/hit_collection.rb
+- lib/relaton/evs/item_data.rb
+- lib/relaton/evs/model/bibdata.rb
+- lib/relaton/evs/model/bibitem.rb
+- lib/relaton/evs/model/docidentifier.rb
+- lib/relaton/evs/model/ext.rb
+- lib/relaton/evs/model/item.rb
+- lib/relaton/evs/model/structured_identifier.rb
+- lib/relaton/evs/processor.rb
+- lib/relaton/evs/scraper.rb
+- lib/relaton/evs/util.rb
 - lib/relaton/gb.rb
 - lib/relaton/gb/bibdata.rb
 - lib/relaton/gb/bibitem.rb