relaton 3.0.0.pre.alpha.14 → 3.0.0.pre.alpha.16

data/lib/relaton/evs/model/docidentifier.rb

@@ -0,0 +1,79 @@
+module Relaton
+  module Evs
+    # An EVS document identifier backed by `Pubid::Evs`.
+    #
+    # `content` stays a plain string, so serialization is unchanged; the parsed
+    # identifier lives beside it in `@pubid` and drives the three mutators.
+    # Follows `Relaton::Cen::Docidentifier`. The parse is on the DATA side: it
+    # lazily requires pubid and rescues `LoadError`/`StandardError`, so a
+    # non-EVS value or a code the grammar rejects stays verbatim with `@pubid`
+    # nil, and all three mutators no-op. A malformed *query* raises — see
+    # `Bibliography.parse`.
+    class Docidentifier < Bib::Docidentifier
+      # @return [Pubid::Evs::Identifier, nil]
+      attr_reader :pubid
+
+      # Capture the inherited (LocalizedMarkedUpString) content setter before
+      # overriding #content=, so #refresh_content! writes the re-rendered string
+      # back WITHOUT re-parsing it and discarding the mutation.
+      alias_method :store_content, :content=
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          require "pubid"
+          ::Pubid::Evs.parse(value)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+
+      # Drops the LAST year, and only that one. On a supplement the identifier
+      # names the supplement (`EVS-EN ISO 9001:2015/A1:2024` is amendment A1 of
+      # 2024), so the supplement year goes first and the base document's year
+      # only when there is no supplement year.
+      def remove_date!
+        return unless @pubid
+
+        stripped = @pubid.exclude(:supplement_year)
+        stripped = @pubid.exclude(:year) if stripped == @pubid
+        replace_pubid stripped
+      end
+
+      def remove_part!
+        return unless @pubid
+
+        replace_pubid @pubid.exclude(:part)
+      end
+
+      # Strips the part and the date, then wraps `@pubid` in pubid's `AllParts`
+      # wrapper. `content` stays the plain part-and-date-stripped string on
+      # purpose (same trade-off as `Relaton::Cen::Docidentifier`); `#pubid`
+      # read directly answers `all_parts? == true` and renders WITH pubid's
+      # own "(all parts)" marker.
+      def to_all_parts!
+        return if !@pubid || @pubid.all_parts?
+
+        original = @pubid.exclude
+        remove_part!
+        remove_date!
+        @pubid = original.to_all_parts
+      end
+
+      private
+
+      # `Pubid#exclude` returns a COPY (the BSI/CEN shape), so the new
+      # identifier replaces the old one; it does not mutate in place.
+      def replace_pubid(new_pubid)
+        @pubid = new_pubid
+        refresh_content!
+      end
+
+      def refresh_content!
+        store_content(@pubid.to_s) if @pubid
+      end
+    end
+  end
+end

data/lib/relaton/evs/model/ext.rb

@@ -0,0 +1,12 @@
+require_relative "structured_identifier"
+
+module Relaton
+  module Evs
+    class Ext < Bib::Ext
+      attribute :structuredidentifier, StructuredIdentifier, collection: true, initialize_empty: true
+
+      # No `relaton-model-evs` gem exists yet, so the schema version stays the
+      # `Bib::Ext` default (nil), like the other model-less flavors.
+    end
+  end
+end

data/lib/relaton/evs/model/item.rb

@@ -0,0 +1,14 @@
+require_relative "../item_data"
+require_relative "docidentifier"
+require_relative "ext"
+
+module Relaton
+  module Evs
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/evs/model/structured_identifier.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Evs
+    class StructuredIdentifier < Bib::StructuredIdentifier
+      def remove_date!
+        self.year = nil
+      end
+    end
+  end
+end

data/lib/relaton/evs/processor.rb

@@ -0,0 +1,50 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Evs
+    class Processor < Relaton::Core::Processor
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_evs
+        @prefix = "EVS"
+        @pubid_flavor = :Evs
+        @defaultprefix = %r{^EVS[\s-]}
+        @idtype = "EVS"
+      end
+
+      # @param code [String]
+      # @param date [String, NilClass] year
+      # @param opts [Hash]
+      # @return [Relaton::Evs::ItemData]
+      def get(code, date, opts)
+        require_relative "../evs"
+        ::Relaton::Evs::Bibliography.get(code, date, opts)
+      end
+
+      # `Bibliography.get` applies a year to the document the reference names.
+      def fold_year(pubid, year)
+        fold_year_on_root pubid, year
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Evs::ItemData]
+      def from_xml(xml)
+        require_relative "../evs"
+        ::Relaton::Evs::Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Evs::ItemData]
+      def from_yaml(yaml)
+        require_relative "../evs"
+        ::Relaton::Evs::Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../evs"
+        @grammar_hash ||= ::Relaton::Evs.grammar_hash
+      end
+    end
+  end
+end

data/lib/relaton/evs/scraper.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Evs
+    # Scraper.
+    module Scraper
+      class << self
+        # Parse page.
+        # @param hit [Relaton::Evs::Hit]
+        # @return [Relaton::Evs::ItemData]
+        def parse_page(hit)
+          doc = hit.hit_collection.agent.get hit.hit[:url]
+          id = hit.pubid
+          lang = id&.language == "et" ? "et" : "en"
+          date = fetch_date(doc)
+
+          ItemData.new(
+            fetched: Date.today.to_s,
+            type: "standard",
+            docidentifier: fetch_docid(doc),
+            language: [lang],
+            script: ["Latn"],
+            title: fetch_titles(doc, lang),
+            date: date ? [date] : [],
+            contributor: fetch_contributors,
+            abstract: fetch_abstract(doc, lang),
+            copyright: fetch_copyright(doc, id),
+            source: fetch_source(hit.hit[:url]),
+            relation: fetch_relations(doc),
+            place: [Bib::Place.new(city: "Tallinn")],
+            ext: fetch_ext(doc, hit)
+          )
+        end
+
+        private
+
+        # @param doc [Mechanize::Page]
+        # @return [Array<Relaton::Evs::Docidentifier>]
+        def fetch_docid(doc)
+          code = doc.at_xpath("//h1")&.text&.strip
+          return [] unless code
+
+          [Docidentifier.new(type: "EVS", content: code, primary: true)]
+        end
+
+        # @param doc [Mechanize::Page]
+        # @param lang [String]
+        # @return [Array<Relaton::Bib::Title>]
+        def fetch_titles(doc, lang)
+          title = doc.at_xpath("//h2[contains(@class,'product-title')]")&.text&.strip
+          return [] unless title
+
+          # `from_string` splits on " - " into intro/main and returns an Array.
+          Bib::Title.from_string(title, lang, "Latn")
+        end
+
+        # The "Valid from" badge is the date the document entered into force.
+        #
+        # @param doc [Mechanize::Page]
+        # @return [Relaton::Bib::Date, nil]
+        def fetch_date(doc)
+          badge = doc.at_xpath(
+            "//div[contains(@class,'product-info-badge')][contains(., 'Valid from')]",
+          )
+          return unless badge
+
+          d = badge.text.match(/(\d{2})\.(\d{2})\.(\d{4})/)
+          return unless d
+
+          Bib::Date.new(type: "published", at: "#{d[3]}-#{d[2]}-#{d[1]}")
+        end
+
+        # @return [Array<Relaton::Bib::Contributor>]
+        def fetch_contributors
+          role = Bib::Contributor::Role.new(type: "publisher")
+          [Bib::Contributor.new(role: [role], organization: evs_org)]
+        end
+
+        # @param doc [Mechanize::Page]
+        # @param lang [String]
+        # @return [Array<Relaton::Bib::Abstract>]
+        def fetch_abstract(doc, lang)
+          scope = doc.at_xpath("//div[@id='tab-1']")&.text&.strip
+          return [] unless scope
+
+          [Bib::Abstract.new(content: scope, language: lang, script: "Latn")]
+        end
+
+        # @param doc [Mechanize::Page]
+        # @param id [Pubid::Evs::Identifier, nil]
+        # @return [Array<Bib::Copyright>]
+        def fetch_copyright(doc, id)
+          from = id&.root&.year&.to_s
+          from ||= fetch_date(doc)&.at&.to_s&.slice(0, 4)
+          return [] unless from
+
+          owner = Bib::ContributionInfo.new(organization: evs_org)
+          [Bib::Copyright.new(owner: [owner], from: from)]
+        end
+
+        # @param url [String]
+        # @return [Array<Relaton::Bib::Uri>]
+        def fetch_source(url)
+          [Bib::Uri.new(type: "src", content: url)]
+        end
+
+        # The "Base Documents" group names the documents this record adopts:
+        # the European document for a national adoption (`EN 18216:2026`), the
+        # international document for a direct adoption. A double adoption
+        # lists both in one text node ("ISO 14001:2026; EN ISO 14001:2026"),
+        # so the pieces split on the semicolon. Each becomes an `adoptedFrom`
+        # relation.
+        #
+        # @param doc [Mechanize::Page]
+        # @return [Array<Relaton::Bib::Relation>]
+        def fetch_relations(doc)
+          group = doc.at_xpath(
+            "//div[contains(@class,'product-ics-group')]" \
+            "[div[contains(., 'Base Documents')]]",
+          )
+          return [] unless group
+
+          bases(group).map do |base|
+            Bib::Relation.new(
+              type: "adoptedFrom",
+              bibitem: ItemData.new(
+                formattedref: Bib::Formattedref.new(content: base),
+                type: "standard",
+              ),
+            )
+          end
+        end
+
+        def bases(group)
+          group.xpath("./a | ./text()").map(&:text)
+               .flat_map { |t| t.split(";") }.map(&:strip)
+               .reject(&:empty?).uniq
+        end
+
+        # @param doc [Mechanize::Page]
+        # @return [Array<Relaton::Bib::ICS>]
+        def fetch_ics(doc)
+          doc.xpath(
+            "//div[contains(@class,'product-group-heading')][contains(., 'ICS')]",
+          ).each_with_object([]) do |heading, acc|
+            heading.parent.xpath("./a").each do |a|
+              code = a.text.strip[/^[\d.]+/]
+              next if code.nil? || acc.any? { |ics| ics.code == code }
+
+              isoics = Isoics.fetch code
+              acc << Bib::ICS.new(code: code, text: isoics.description)
+            end
+          end
+        end
+
+        # @param doc [Mechanize::Page]
+        # @param hit [Relaton::Evs::Hit]
+        # @return [Relaton::Evs::Ext]
+        def fetch_ext(doc, hit)
+          Ext.new(
+            doctype: fetch_doctype(hit.pubid),
+            flavor: "evs",
+            ics: fetch_ics(doc),
+            structuredidentifier: fetch_structuredid(hit.pubid),
+          )
+        end
+
+        # The shared ext grammar requires a doctype. The TS and TR faces of a
+        # plain Estonian standard map to their document types; every other
+        # form is a standard.
+        #
+        # @param id [Pubid::Evs::Identifier, nil]
+        # @return [Relaton::Bib::Doctype]
+        def fetch_doctype(id)
+          evs_type = id&.evs_type if id&.respond_to?(:evs_type)
+          case evs_type
+          when "TS" then Bib::Doctype.new(content: "technical-specification")
+          when "TR" then Bib::Doctype.new(content: "technical-report")
+          else Bib::Doctype.new(content: "standard")
+          end
+        end
+
+        # `#root` is the accessor that answers for every form: a national
+        # adoption keeps the origin standard's number and part behind its
+        # `base` chain.
+        #
+        # @param id [Pubid::Evs::Identifier, nil]
+        # @return [Relaton::Evs::StructuredIdentifier]
+        def fetch_structuredid(id)
+          root = id&.root
+          StructuredIdentifier.new(
+            docnumber: root&.number&.to_s,
+            partnumber: root&.part.to_s,
+            agency: ["EVS"],
+          )
+        end
+
+        # @return [Hash]
+        def evs_org
+          @evs_org ||= begin
+            name = Bib::TypedLocalizedString.new(
+              content: "Eesti Standardimis- ja Akrediteerimiskeskus",
+            )
+            abbr = Bib::TypedLocalizedString.new(content: "EVS")
+            uri = Bib::Uri.new(content: "https://www.evs.ee")
+            Bib::Organization.new(name: [name], abbreviation: abbr, uri: [uri])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/relaton/evs/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Evs
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-evs".freeze
+    end
+  end
+end

data/lib/relaton/evs.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "digest/md5"
+require "mechanize"
+require "relaton/core"
+require "isoics"
+require "pubid"
+require "relaton/bib"
+require_relative "version"
+require_relative "evs/util"
+require_relative "evs/model/item"
+require_relative "evs/model/bibitem"
+require_relative "evs/model/bibdata"
+require_relative "evs/scraper"
+require_relative "evs/hit_collection"
+require_relative "evs/hit"
+require_relative "evs/bibliography"
+
+module Relaton
+  module Evs
+    # Returns hash of XML grammar
+    # @return [String]
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/gost/bibliography.rb

@@ -73,10 +73,13 @@ module Relaton
         private
 
         def index
+          # The machine index on the Pages site (relaton#206): a parsed query
+          # reads only its own shard; a Pages failure raises
+          # `Relaton::RequestError`; the whole-index zip on the same site
+          # serves only the retry on a miss.
           Relaton::Index.find_or_create(
             :gost,
-            url: "#{ENDPOINT}#{INDEXFILE}.zip",
-            file: "#{INDEXFILE}.yaml",
+            pages_url: PAGES_URL,
             pubid_class: ::Pubid::Gost::Identifier,
           )
         end

data/lib/relaton/gost.rb

@@ -12,6 +12,10 @@ require "relaton/version"
 module Relaton
   module Gost
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-gost/"
 
     class Error < StandardError; end
 

data/lib/relaton/iala/bibliography.rb

@@ -136,10 +136,13 @@ module Relaton
         # `id.root.number`, and let `Type#search` bsearch. Without it the rows
         # stay raw hashes and every lookup scans all 701 of them.
         def index
+          # The machine index on the Pages site (relaton#206): a parsed query
+          # reads only its own shard; a Pages failure raises
+          # `Relaton::RequestError`; the whole-index zip on the same site
+          # serves only the retry on a miss.
           Relaton::Index.find_or_create(
             :iala,
-            url: "#{ENDPOINT}#{INDEXFILE}.zip",
-            file: "#{INDEXFILE}.yaml",
+            pages_url: PAGES_URL,
             pubid_class: ::Pubid::Iala::Identifier,
           )
         end

data/lib/relaton/iala.rb

@@ -10,6 +10,10 @@ require "relaton/version"
 module Relaton
   module Iala
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-iala/"
 
     class Error < StandardError; end
 

data/lib/relaton/iana/bibliography.rb

@@ -53,8 +53,11 @@ module Relaton
       private
 
       def index
-        Relaton::Index.find_or_create :iana, url: "#{SOURCE}#{INDEXFILE}.zip",
-                                             file: "#{INDEXFILE}.yaml",
+        # The machine index on the Pages site (relaton#206): a parsed query
+        # reads only its own shard; a Pages failure raises
+        # `Relaton::RequestError`; the whole-index zip on the same site
+        # serves only the retry on a miss.
+        Relaton::Index.find_or_create :iana, pages_url: PAGES_URL,
                                              pubid_class: ::Pubid::Iana::Identifier
       end
 

data/lib/relaton/iana.rb

@@ -27,6 +27,10 @@ module Relaton
     # emitted by `relaton-data-iana`'s own `crawler.rb`, derived from `index-v2`,
     # so released gem lines keep resolving. See lib/relaton/iana/CLAUDE.md.
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-iana/"
 
     # Returns hash of XML reammar
     # @return [String]

data/lib/relaton/iec/hit_collection.rb

@@ -47,11 +47,16 @@ module Relaton
         [pubid.part.to_s.to_i, sub.to_s.to_i]
       end
 
+      #
+      # The machine index on the Pages site (relaton#206, the W3C pilot's
+      # contract): a parsed query reads only its own shard. A Pages failure
+      # raises `Relaton::RequestError`; the whole-index zip on the same site
+      # serves only the retry on a miss.
+      #
       def index
         @index ||= Relaton::Index.find_or_create(
           :IEC,
-          url: "#{Hit::GHURL}#{INDEXFILE}.zip",
-          file: "#{INDEXFILE}.yaml",
+          pages_url: PAGES_URL,
           pubid_class: ::Pubid::Iec::Identifier
         )
       end

data/lib/relaton/iec.rb

@@ -21,6 +21,10 @@ require_relative "iec/model/bibdata"
 module Relaton
   module Iec
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by HitCollection's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-iec/"
 
     class << self
       # Returns hash of XML reammar

data/lib/relaton/ieee/bibliography.rb

@@ -13,7 +13,11 @@ module Relaton
         # @return [Relaton::Ieee::ItemData, nil]
         #
         def search(ref) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-          index = Relaton::Index.find_or_create :ieee, url: "#{GH_URL}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+          # The machine index on the Pages site (relaton#206, the W3C
+          # pilot's contract): a parsed query reads only its own shard. A
+          # Pages failure raises `Relaton::RequestError`; the whole-index
+          # zip on the same site serves only the retry on a miss.
+          index = Relaton::Index.find_or_create :ieee, pages_url: PAGES_URL,
                                                        pubid_class: ::Pubid::Ieee::Identifier
           # Pass the parsed pubid (not the raw String) so index-v2 narrows
           # candidates by number via binary search before the block runs; the

data/lib/relaton/ieee.rb

@@ -17,6 +17,10 @@ module Relaton
     class Error < StandardError; end
 
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by Bibliography's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-ieee/"
 
     # Returns hash of XML reammar
     # @return [String]

data/lib/relaton/ietf/scraper.rb

@@ -47,10 +47,15 @@ module Relaton
         get_page "#{IETF}#{row[:file]}" if row
       end
 
+      #
+      # The machine index on the Pages site (relaton#206, the W3C pilot's
+      # contract): a parsed query reads only its own shard. A Pages failure
+      # raises `Relaton::RequestError`; the whole-index zip on the same site
+      # serves only the case-insensitive retry on a miss.
+      #
       def index
         Relaton::Index.find_or_create(
-          :IETF, url: "#{IETF}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
-                 pubid_class: ::Pubid::Ietf::Identifier
+          :IETF, pages_url: PAGES_URL, pubid_class: ::Pubid::Ietf::Identifier
         )
       end
 

data/lib/relaton/ietf.rb

@@ -18,6 +18,10 @@ module Relaton
     # (relaton#109). One constant again now that both sides are on it — the
     # second one existed only while producer and consumer were split.
     INDEXFILE = "index-v2".freeze
+    # The machine index on the data repo's Pages site (relaton#189, #206) —
+    # manifest + per-shard rows read by Scraper's index; the zip stays on the
+    # Pages site for the whole-index retry on a case-insensitive miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-ietf/".freeze
     # Returns hash of XML reammar
     # @return [String]
     def self.grammar_hash

data/lib/relaton/iho/bibliography.rb

@@ -85,10 +85,13 @@ module Relaton
         private
 
         def index
+          # The machine index on the Pages site (relaton#206): a parsed query
+          # reads only its own shard; a Pages failure raises
+          # `Relaton::RequestError`; the whole-index zip on the same site
+          # serves only the retry on a miss.
           Relaton::Index.find_or_create(
             :iho,
-            url: "#{ENDPOINT}#{INDEXFILE}.zip",
-            file: "#{INDEXFILE}.yaml",
+            pages_url: PAGES_URL,
             pubid_class: ::Pubid::Iho::Identifier,
           )
         end

data/lib/relaton/iho/processor.rb

@@ -3,6 +3,9 @@ require "relaton/core/processor"
 module Relaton
   module Iho
     class Processor < Core::Processor
+      def cache_key(ref, year, opts, parsed = nil)
+        super
+      end
       attr_reader :idtype
 
       def initialize

data/lib/relaton/iho.rb

@@ -12,6 +12,10 @@ require_relative "iho/bibliography"
 module Relaton
   module Iho
     INDEXFILE = "index-v3".freeze
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by the consumer's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-iho/"
 
     class Error < StandardError; end
 

data/lib/relaton/iso/hit_collection.rb

@@ -148,11 +148,16 @@ module Relaton
         pubid.typed_stage.stage_code.to_s == "published"
       end
 
+      #
+      # The machine index on the Pages site (relaton#206, the W3C pilot's
+      # contract): a parsed query reads only its own shard. A Pages failure
+      # raises `Relaton::RequestError`; the whole-index zip on the same site
+      # serves only the retry on a miss.
+      #
       def index
         @index ||= Relaton::Index.find_or_create(
           :iso,
-          url: "#{ENDPOINT}#{INDEXFILE}.zip",
-          file: "#{INDEXFILE}.yaml",
+          pages_url: PAGES_URL,
           pubid_class: ::Pubid::Iso::Identifier,
         )
       end

data/lib/relaton/iso.rb

@@ -19,6 +19,10 @@ require_relative "iso/bibliography"
 module Relaton
   module Iso
     INDEXFILE = "index-v2"
+    # The machine index on the data repo's Pages site (relaton#206) —
+    # manifest + per-shard rows read by HitCollection's index; the zip stays
+    # on the Pages site for the whole-index retry on a miss.
+    PAGES_URL = "https://relaton.github.io/relaton-data-iso/"
 
     def self.grammar_hash
       # gem_path = File.expand_path "..", __dir__