relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/xsf/bibliography.rb

@@ -3,8 +3,36 @@ module Relaton
     module Bibliography
       extend self
 
+      #
+      # Search the index for a reference.
+      #
+      # The reference is parsed into a `Pubid::Xsf::Identifier` here and the
+      # identifier -- never the string -- is what reaches `HitCollection`.
+      # `Index::Type#search_candidates` narrows only when its argument is not a
+      # `String`, so parsing at the entry point is what lets the lookup binary
+      # search; `Core::HitCollection` takes a `String` or a pubid by design.
+      #
+      # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
+      # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
+      # `"..." is not a recognized standards identifier`
+      # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`,
+      # `subcommand_collection.rb:134`), and `Db#fetch` logs it via the
+      # `StandardError` arm at `lib/relaton/db.rb:122`. Rescuing here would
+      # collapse "this identifier is malformed" into "no such document", and a
+      # caller could no longer tell them apart.
+      #
+      # There is also deliberately no substring fallback: the old
+      # `index.search(ref)` compared a substring of the rendered id, so a bare
+      # `001` answered with 11 documents and `#get` took `.first` -- a truncated
+      # reference silently resolved to whichever sorted first.
+      #
+      # @param ref [String] e.g. "XEP 0001", "XEP-0001", "0001"
+      #
+      # @return [Relaton::Xsf::HitCollection]
+      # @raise [Pubid::Errors::ParseError] if the reference is not an XEP id
+      #
       def search(ref)
-        HitCollection.new(ref).search
+        HitCollection.new(parse_ref(ref)).search
       end
 
       def get(code, _year = nil, _opts = {})
@@ -19,6 +47,38 @@ module Relaton
         Util.info "Found: `#{bib.docidentifier.first.content}`", key: code
         bib
       end
+
+      #
+      # Parse a user reference into a `Pubid::Xsf::Identifier`, or nil.
+      #
+      # `Pubid::Xsf` accepts only the canonical `XEP 0001` spelling, so two
+      # forms are normalized first:
+      #
+      # - **`XEP-0001`**, the spelling xmpp.org itself uses everywhere. It did
+      #   not resolve before either (the substring match compared against
+      #   `XEP 0001`, which has a space), so this is new support rather than a
+      #   preserved behaviour.
+      # - **A bare `0001`**, which the substring match did resolve, so it has to
+      #   keep working. The publisher token is added, the W3C idiom.
+      #
+      # The token match is case-insensitive, so `xep 0001` resolves too.
+      #
+      # Anything else raises -- see `.search`. `Pubid::Errors::ParseError`
+      # includes `Pubid::Errors::Error`, which is the class relaton-cli rescues.
+      #
+      # @param ref [String]
+      # @return [Pubid::Xsf::Identifier]
+      # @raise [Pubid::Errors::ParseError]
+      #
+      def parse_ref(ref)
+        ::Pubid::Xsf::Identifier.parse normalize_ref(ref)
+      end
+
+      private
+
+      def normalize_ref(ref)
+        "XEP #{ref.to_s.strip.sub(/\AXEP[-\s]+/i, '')}"
+      end
     end
   end
 end

data/lib/relaton/xsf/data_fetcher.rb

@@ -3,8 +3,19 @@ require "relaton/core"
 module Relaton
   module Xsf
     class DataFetcher < Relaton::Core::DataFetcher
+      # `Core::DataFetcher#log_error` raises unless the flavor overrides it, so
+      # `report_errors` is unusable without this. (The ECMA precedent.)
+      def log_error(msg)
+        Util.error msg
+      end
+
       def index
-        @index ||= Relaton::Index.find_or_create :xsf, file: "#{INDEXFILE}.yaml"
+        # `pubid_class:` on the producer too: FileIO#save only calls `to_hash`
+        # when the value is an instance of it, so without it the crawl writes
+        # v1-shaped rows under a v2 name, silently.
+        @index ||= Relaton::Index.find_or_create(
+          :xsf, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Xsf::Identifier
+        )
       end
 
       def fetch(_source = nil)
@@ -18,6 +29,7 @@ module Relaton
           Util.warn "Failed to parse #{link[:href]}: #{e.message}"
         end
         index.save
+        report_errors
       end
 
       def save_doc(bib)
@@ -30,14 +42,52 @@ module Relaton
         id = docid&.content
         return unless id
 
-        file = output_file id
+        # Distinct docids can sanitize to one filename; take a path of our own
+        # rather than overwriting the other document (Core#unique_output_file).
+        file = unique_output_file id
         if @files.include? file
+          # Same reserved path == same id: a genuine duplicate. Checked FIRST,
+          # because a disambiguated path stays != output_file forever.
           Util.warn "File #{file} already exists"
-        else
-          @files << file
+        elsif file != output_file(id)
+          Util.warn "File #{output_file id} already exists; writing #{file} instead"
         end
+        @files << file
         File.write file, serialize(bib), encoding: "UTF-8"
-        index.add_or_update id, file
+        add_to_index id, file
+      end
+
+      #
+      # Index a document under its pubid, skipping one pubid cannot parse.
+      #
+      # This guard is not optional here. One unparseable row does not fail that
+      # row: `Relaton::Index` declares the **whole file** corrupt, deletes it,
+      # and hands back an **empty** index. Measured — 519 good rows plus one
+      # unparseable id loads as 0 rows, and the only trace is two INFO lines
+      # ("Wrong structure of file …", "Considering … corrupt, removing it"). So
+      # indexing one bad row silently breaks every XSF lookup, not just its own.
+      #
+      # A rejection is recorded in `@errors`, which `report_errors` turns into a
+      # GitHub issue at the end of the crawl (the 3GPP/ECMA precedent). Nothing
+      # in the published corpus trips it today: pubid parses all 520 rows,
+      # including the two entries that are pages rather than XEPs — the XEP
+      # repository's `README` and its `xep-xxxx` template, which pubid accepts
+      # as the literal numbers `README` and `xxxx` while still rejecting
+      # anything else non-numeric (`XEP banana`, or a typo like `XEP 00O1`).
+      # So a recorded error now means something genuinely new appeared, which
+      # is exactly when an issue is worth filing.
+      #
+      # The data file is written either way, so a document that cannot be
+      # indexed is unindexed, never lost.
+      #
+      # @param docid [String]
+      # @param file [String]
+      #
+      def add_to_index(docid, file)
+        index.add_or_update ::Pubid::Xsf::Identifier.parse(docid), file
+      rescue StandardError => e
+        @errors[docid] =
+          "Unparseable primary id `#{docid}` was not indexed (#{e.message})"
       end
 
       def to_yaml(bib)

data/lib/relaton/xsf/docidentifier.rb

@@ -0,0 +1,46 @@
+module Relaton
+  module Xsf
+    # An XSF document identifier, backed by `Pubid::Xsf`.
+    #
+    # Parses its `content` into a `Pubid::Xsf::Identifier` kept in `@pubid`,
+    # while the lutaml `content` attribute stays a plain **string** for
+    # serialization. Parsing is **soft**: `content=` lazily requires pubid and
+    # rescues `LoadError`/`StandardError`, so a missing gem or non-XSF content
+    # leaves `@pubid` nil rather than raising during deserialization.
+    #
+    # **It deliberately implements none of `remove_part!` / `remove_date!` /
+    # `to_all_parts!`.** An XEP identifier is a publisher and a number and
+    # nothing else — no part, no edition, no date — so there is genuinely
+    # nothing for any of them to strip, and `Bib::Docidentifier` already
+    # defaults all three to no-ops for exactly that case. Empty overrides here
+    # would assert a flavor-specific rule that does not exist. Compare
+    # `Relaton::Ogc::Docidentifier`, which overrides `remove_date!` because OGC
+    # really does carry a revision.
+    #
+    # So this class exists for `#pubid`: it gives consumers the structured
+    # identifier off a docidentifier, the way every other pubid-backed flavor
+    # does, without changing what is serialized.
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      def initialize(attrs = {}, options = {})
+        pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
+        attrs[:content] ||= pubid.to_s if pubid
+        super
+        @pubid = pubid if pubid
+      end
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          require "pubid"
+          ::Pubid::Xsf::Identifier.parse(value)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/relaton/xsf/hit_collection.rb

@@ -3,11 +3,29 @@ module Relaton
     class HitCollection < Relaton::Core::HitCollection
       GHDATA_URL = "https://raw.githubusercontent.com/relaton/relaton-data-xsf/v2/".freeze
 
+      #
+      # Find the index rows for the identifier this collection was built with.
+      #
+      # `ref` is a `Pubid::Xsf::Identifier` -- `Bibliography.parse_ref` does that
+      # step, and raises rather than returning nil for an unrecognized
+      # reference. Passing the identifier rather than a string is what enables
+      # the binary search on `id.root.number`. The nil guard in `#rows` is
+      # defensive only: `Core::HitCollection` permits a nil `ref`, and an empty
+      # result beats a NoMethodError if some other caller constructs one.
+      #
+      # An XEP identifier is only a publisher and a number: no edition, no date,
+      # no part. So the reference states everything, and `Type#search`'s default
+      # subset match (`Pubid::SubsetMatch`) is a plain equality -- which is the
+      # point. Ids are unique, one row per XEP, so there is no selection order
+      # to apply either.
+      #
+      # @return [self]
+      #
       def search
-        @array = index.search(ref).sort_by { |hit| hit[:id] }.map do |row|
-          Hit.new url: "#{GHDATA_URL}#{row[:file]}"
-        end
+        @array = rows.map { |row| Hit.new url: "#{GHDATA_URL}#{row[:file]}" }
         self
+      rescue Relaton::RequestError
+        raise
       rescue StandardError => e
         raise Relaton::RequestError, e.message
       end
@@ -17,8 +35,18 @@ module Relaton
           :xsf,
           url: "#{GHDATA_URL}#{INDEXFILE}.zip",
           file: "#{INDEXFILE}.yaml",
+          pubid_class: ::Pubid::Xsf::Identifier,
         )
       end
+
+      private
+
+      def rows
+        return [] unless ref
+
+        index.search(ref)
+          .sort_by { |row| row[:id].to_s }
+      end
     end
   end
 end

data/lib/relaton/xsf/item.rb

@@ -1,7 +1,13 @@
+require_relative "docidentifier"
+
 module Relaton
   module Xsf
     class Item < Bib::Item
       model ItemData
+
+      # Narrow the inherited `docidentifier` to the flavor's own class, so a
+      # deserialized item exposes `#pubid` on each id. (The OGC/ECMA shape.)
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
     end
   end
 end

data/lib/relaton/xsf/processor.rb

@@ -12,6 +12,7 @@ module Relaton
         @defaultprefix = %r{^XEP\s}
         @idtype = "XEP"
         @datasets = %w[xep-xmpp]
+        @pubid_flavor = :Xsf
       end
 
       def get(code, date, opts)

data/lib/relaton/xsf.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 require "mechanize"
+# pubid loads with the flavor, for the ::Pubid::Xsf::Identifier that the
+# index code names. Processor#remove_index_file names no pubid class: the
+# delete never reads the index (see lib/relaton/index/CLAUDE.md).
+require "pubid"
 require "relaton/index"
 require "relaton/bib"
 require_relative "version"
@@ -15,7 +19,7 @@ require_relative "xsf/bibliography"
 
 module Relaton
   module Xsf
-    INDEXFILE = "index-v1"
+    INDEXFILE = "index-v2".freeze
 
     class Error < StandardError; end
 

data/lib/relaton.rb

@@ -12,6 +12,7 @@ module Relaton
   autoload :Index, "relaton/index"
   autoload :Bib, "relaton/bib"
   autoload :ThreeGpp, "relaton/3gpp"
+  autoload :Adobe, "relaton/adobe"
   autoload :Bipm, "relaton/bipm"
   autoload :Calconnect, "relaton/calconnect"
   autoload :Ccsds, "relaton/ccsds"
@@ -19,13 +20,17 @@ module Relaton
   autoload :Cie, "relaton/cie"
   autoload :Ecma, "relaton/ecma"
   autoload :Etsi, "relaton/etsi"
+  autoload :Easc, "relaton/easc"
+  autoload :Gost, "relaton/gost"
   autoload :Iana, "relaton/iana"
+  autoload :Iala, "relaton/iala"
   autoload :Ieee, "relaton/ieee"
   autoload :Ietf, "relaton/ietf"
   autoload :Iho, "relaton/iho"
   autoload :Isbn, "relaton/isbn"
   autoload :Iso, "relaton/iso"
   autoload :Itu, "relaton/itu"
+  autoload :Jcgm, "relaton/jcgm"
   autoload :Nist, "relaton/nist"
   autoload :Oasis, "relaton/oasis"
   autoload :Oiml, "relaton/oiml"
@@ -40,6 +45,43 @@ module Relaton
   autoload :Ogc, "relaton/ogc"
   autoload :Plateau, "relaton/plateau"
   autoload :Doi, "relaton/doi"
+  autoload :Sdo, "relaton/sdo"
+
+  # Global PubID prefix register (relaton-db#103). Maps an SDO document-ID
+  # prefix to the flavor module(s) that own it. Conflicting prefixes (e.g.
+  # "ISO/IEC", claimed by both ISO and IEC) return several flavors.
+  #
+  #   Relaton.prefix_flavor("NIST")    # => [Relaton::Nist]
+  #   Relaton.prefix_flavor("ISO/IEC") # => [Relaton::Iec, Relaton::Iso]
+  #   Relaton.prefix_flavor("BOGUS")   # => []
+  #
+  # NOTE: returning the flavor module forces that flavor's lazy load. To resolve
+  # a prefix without loading flavor code, use
+  # `Relaton::Db::Registry.instance.processors_by_prefix(prefix)`.
+  #
+  # @param prefix [String]
+  # @return [Array<Module>]
+  def self.prefix_flavor(prefix)
+    Db::Registry.instance.flavors_by_prefix(prefix)
+  end
+
+  # SDO organization & logo store (metanorma#346, relaton-db#132). Looks up a
+  # standards body by abbreviation and returns its metadata — names/translations
+  # and logo variants — from the `relaton-data-sdo` data repo. This is a
+  # non-flavor store: no processor, no Db::Registry entry.
+  #
+  #   org = Relaton.organization("ISO")
+  #   org.name                                   # default name
+  #   org.name("fr")                             # translated name
+  #   org.name(language: "fr")                   # translated name (keyword form)
+  #   org.logo_query(format: "eps", style: "default")  # matching logo variants
+  #   org.logo(format: "eps", style: "default")        # one logo
+  #
+  # @param abbreviation [String]
+  # @return [Relaton::Sdo::Organization, nil]
+  def self.organization(abbreviation)
+    Sdo::Store.instance.organization(abbreviation)
+  end
 end
 
 require "relaton/db"