relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/oasis/processor.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "relaton/core/processor"
+
+module Relaton
+  module Oasis
+    class Processor < Core::Processor
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_oasis
+        @prefix = "OASIS"
+        @defaultprefix = %r{^OASIS\s}
+        @idtype = "OASIS"
+        @datasets = %w[oasis-open]
+        @pubid_flavor = :Oasis
+      end
+
+      # @param code [String]
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [Relaton::Oasis::Item]
+      def get(code, date, opts)
+        require_relative "../oasis"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Fetch all the documents from https://www.oasis-open.org/standards/
+      #
+      # @param [String] _source source name
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format
+      #
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Oasis::Item]
+      def from_xml(xml)
+        require_relative "../oasis"
+        Item.from_xml(xml)
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Oasis::Item]
+      def from_yaml(yaml)
+        require_relative "../oasis"
+        Item.from_yaml(yaml)
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../oasis"
+        @grammar_hash ||= Relaton::Oasis.grammar_hash
+      end
+
+      #
+      # Remove index file
+      #
+      # `url: true` names the cached file. No `pubid_class:`: `Type#remove_file`
+      # deletes the file and never reads the index.
+      #
+      def remove_index_file
+        require_relative "../oasis"
+        Relaton::Index.find_or_create(
+          :oasis, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/oasis/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Oasis
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-oasis".freeze
+    end
+  end
+end

data/lib/relaton/oasis.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Not lazy: DataFetcher names ::Pubid::Oasis::Identifier as the index
+# `pubid_class:`, and Docidentifier parses every docid through it.
+# (The IANA/IHO/IALA/OGC/ECMA form; spec/relaton/lazy_loading_spec.rb guards
+# that this file is not itself loaded when a Db is built.)
+require "pubid"
+require "relaton/index"
+require "relaton/bib"
+require_relative "version"
+require_relative "oasis/util"
+require_relative "oasis/item_data"
+require_relative "oasis/item"
+require_relative "oasis/bibitem"
+require_relative "oasis/bibdata"
+require_relative "oasis/bibliography"
+
+module Relaton
+  module Oasis
+    # The pubid index, both written and read: rows are
+    # `Pubid::Oasis::Identifier` hashes (`_type: pubid:oasis:standard`), built
+    # and read with `pubid_class: ::Pubid::Oasis::Identifier`.
+    #
+    # This gem does not produce `index-v1` any more; `relaton-data-oasis`'s
+    # crawler derives it from the v2 rows for relaton v2 consumers (the
+    # IANA/BIPM/W3C/ECMA shape).
+    INDEXFILE = "index-v2".freeze
+
+    class Error < StandardError; end
+    # Your code goes here...
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/ogc/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ogc
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/ogc/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ogc
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/ogc/bibliography.rb

@@ -0,0 +1,64 @@
+module Relaton
+  module Ogc
+    module Bibliography
+      extend self
+
+      # @param text [String]
+      # @param year [String, nil]
+      # @return [Relaton::Ogc::HitCollection]
+      def search(text, year = nil, _opts = {})
+        code = text.sub(/^OGC\s/, "")
+        HitCollection.new(code, year).find
+      rescue Faraday::ConnectionFailed, Faraday::SSLError
+        raise Relaton::RequestError, HitCollection::ENDPOINT
+      end
+
+      # @param code [String]
+      # @param year [String, nil]
+      # @param opts [Hash]
+      # @return [Relaton::Ogc::ItemData, nil]
+      def get(code, year = nil, opts = {})
+        result = bib_search_filter(code, year, opts) || (return nil)
+        ret = bib_results_filter(result, year)
+        if ret[:ret]
+          Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: code
+          ret[:ret]
+        else
+          fetch_ref_err(code, year, ret[:years])
+        end
+      end
+
+      private
+
+      def bib_search_filter(code, year, opts)
+        Util.info "Fetching from Relaton repository ...", key: code
+        search(code, year, opts)
+      end
+
+      def bib_results_filter(result, year)
+        missed_years = []
+        result.each do |r|
+          item = r.item
+          return { ret: item } unless year
+
+          item.date.select { |d| d.type == "published" }.each do |d|
+            date_year = ::Date.parse(d.at.to_s).year
+            return { ret: item } if year.to_i == date_year
+
+            missed_years << date_year
+          end
+        end
+        { years: missed_years }
+      end
+
+      def fetch_ref_err(code, year, missed_years)
+        Util.info "Not found.", key: code
+        unless missed_years.empty?
+          Util.info "There was no match for `#{year}`, though there " \
+                    "were matches found for `#{missed_years.join('`, `')}`.", key: code
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/relaton/ogc/data_fetcher.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "faraday"
+require "json"
+require_relative "../ogc"
+require_relative "scraper"
+
+module Relaton
+  module Ogc
+    class DataFetcher < Core::DataFetcher
+      ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/NamingAuthority/master/definitions/docs/docs.json"
+
+      def initialize(output, format)
+        super
+        @etagfile = File.join output, "etag.txt"
+        @docids = []
+        @dupids = Set.new
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def index
+        # `pubid_class:` on the producer too: FileIO#save only calls `to_hash`
+        # when the value is an instance of it, so without it the crawl writes
+        # v1-shaped rows under a v2 name, silently.
+        @index ||= Relaton::Index.find_or_create(
+          :ogc, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Ogc::Identifier
+        )
+      end
+
+      def fetch(_source = nil) # rubocop:disable Metrics/AbcSize
+        get_data do |etag, json|
+          no_errors = true
+          json.each_value { |hit| fetch_doc(hit) || no_errors = false }
+          if @dupids.any?
+            Util.warn "Duplicated documents: #{@dupids.to_a.join(', ')}"
+          end
+          self.etag = etag if no_errors
+          index.save
+          report_errors
+        end
+      end
+
+      def fetch_doc(hit)
+        return if hit["type"] == "CC"
+
+        bib = Scraper.parse_page hit, @errors
+        write_document bib
+        true
+      rescue StandardError => e
+        Util.error "Fetching document: #{hit['identifier']}\n" \
+                   "#{e.class} #{e.message}\n#{e.backtrace}"
+        false
+      end
+
+      def write_document(bib) # rubocop:disable Metrics/AbcSize
+        docid = bib.docidentifier[0].content
+        if @docids.include?(docid)
+          @dupids << docid
+          return
+        end
+
+        @docids << docid
+        file = file_name bib
+        index.add_or_update index_id(docid), file
+        File.write file, serialize(bib), encoding: "UTF-8"
+      end
+
+      #
+      # The index key. Stored as a `Pubid::Ogc::Identifier` so the published
+      # index-v2 rows are pubid hashes the consumer can narrow on. A docid
+      # pubid rejects is logged and indexed as the plain string, so one bad
+      # record never aborts a crawl.
+      #
+      # @param docid [String]
+      # @return [Pubid::Ogc::Identifier, String]
+      #
+      def index_id(docid)
+        ::Pubid::Ogc::Identifier.parse docid
+      rescue StandardError => e
+        Util.warn "Failed to parse pubid `#{docid}`: #{e.message}"
+        docid
+      end
+
+      def file_name(bib)
+        name = bib.docidentifier[0].content.upcase.gsub(/[\s:.]/, "_")
+        "#{@output}/#{name}.#{@ext}"
+      end
+
+      def to_yaml(bib)
+        Item.to_yaml bib
+      end
+
+      def to_xml(bib)
+        Bibdata.to_xml bib
+      end
+
+      def to_bibxml(_bib)
+        raise NotImplementedError, "OGC does not support bibxml format"
+      end
+
+      # @return [String, nil]
+      def etag
+        @etag ||= if File.exist? @etagfile
+                    File.read @etagfile, encoding: "UTF-8"
+                  end
+      end
+
+      # @param e_tag [String]
+      def etag=(e_tag)
+        File.write @etagfile, e_tag, encoding: "UTF-8"
+      end
+
+      def get_data # rubocop:disable Metrics/AbcSize
+        h = {}
+        h["If-None-Match"] = etag if etag
+        resp = Faraday.new(ENDPOINT, headers: h).get
+        case resp.status
+        when 200
+          json = JSON.parse(resp.body)
+          block_given? ? yield(resp[:etag], json) : json
+        when 304 then []
+        else raise Relaton::RequestError, "Could not access #{ENDPOINT}"
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ogc/docidentifier.rb

@@ -0,0 +1,111 @@
+module Relaton
+  module Ogc
+    # Structured OGC document identifier. The docid string is the canonical
+    # printed form, which carries no publisher token (e.g. "19-025r1"), and it
+    # is parsed into a `Pubid::Ogc::Identifier` kept in `@pubid` while the
+    # lutaml `content` attribute stays a plain string for serialization.
+    #
+    # Parsing is **soft**: `content=` lazily requires pubid and rescues
+    # `LoadError`/`StandardError`, so a missing pubid gem or non-OGC content
+    # leaves `@pubid` nil rather than raising.
+    #
+    # Before this class had a body it inherited `Bib::Docidentifier`, whose
+    # `remove_part!` / `remove_date!` / `to_all_parts!` each raise
+    # `NotImplementedError` — and `Bib::ItemData` broadcasts all three to every
+    # docidentifier, so `#to_all_parts` and `#to_most_recent_reference` raised
+    # on **every** OGC item. `NotImplementedError` descends from `ScriptError`,
+    # so a caller's `rescue => e` did not even catch it.
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      # Capture the inherited (LocalizedMarkedUpString) content setter before
+      # overriding #content=, so #refresh_content! can write the re-rendered
+      # string back WITHOUT re-parsing (a re-parse would rebuild @pubid from the
+      # string and discard in-place mutations such as `all_parts = true`).
+      alias_method :store_content, :content=
+
+      def initialize(attrs = {}, options = {})
+        pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
+        attrs[:content] ||= pubid.to_s if pubid
+        super
+        @pubid = pubid if pubid
+      end
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          require "pubid"
+          ::Pubid::Ogc::Identifier.parse(value)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+
+      # `Pubid::Ogc::Identifier` models only `year`/`number`/`revision`, so the
+      # mapping is OGC-specific:
+      #
+      # - **`remove_date!` → clears `revision`.** OGC carries no date component;
+      #   `revision` is its version discriminator — the same component a
+      #   reference may omit, which `HitCollection#best_match` then treats as a
+      #   wildcard — so clearing it yields
+      #   the version-agnostic ("most recent") reference: `12-128r19` →
+      #   `12-128`.
+      # - **`year` is never cleared.** It looks date-like but is half the
+      #   document number (`12-128`), not a publication qualifier; dropping it
+      #   would leave `-128`, which identifies nothing.
+      # - **`remove_part!` → clears the (unused) `part`/`subpart` attributes.**
+      #   A no-op for the rendered string today: OGC has no part component.
+      #   Implemented anyway so it never raises, and it starts working
+      #   automatically if pubid-ogc ever models one. (The IALA precedent.)
+      # - **`to_all_parts!` → both, then wraps `@pubid` in pubid's `AllParts`.**
+      #   `content` stays the plain stripped id — OGC's own renderer has no
+      #   "(all parts)" marker, and `content` is cached from the pre-wrap
+      #   pubid on purpose. `#pubid` itself, read directly, now answers
+      #   `all_parts? == true` and renders WITH pubid's generic marker
+      #   (`#to_s`), since it's the wrapper.
+      #
+      # All three no-op safely when `@pubid` is nil, so `Bib::ItemData`'s
+      # `#to_all_parts` / `#to_most_recent_reference` never raise on OGC items.
+      #
+      # `refresh_content!` renders a bare `to_s`: the OGC printed form has no
+      # publisher token and the stored `content` does not carry one either, so
+      # unlike 3GPP this must NOT pass `with_publisher: true`.
+
+      def remove_part!
+        clear_attr!(:part)
+      end
+
+      def remove_date!
+        clear_attr!(:revision)
+      end
+
+      def to_all_parts!
+        return if !@pubid || @pubid.all_parts?
+
+        # `#exclude` (no args) rebuilds a full independent copy — `remove_part!`
+        # / `remove_date!` mutate `@pubid` in place, so a bare reference here
+        # would lose the original revision to that mutation too.
+        original = @pubid.exclude
+        remove_part!
+        remove_date!
+        @pubid = original.to_all_parts
+      end
+
+      private
+
+      def clear_attr!(attr)
+        return unless @pubid && @pubid.respond_to?("#{attr}=")
+
+        @pubid.public_send("#{attr}=", nil)
+        @pubid.subpart = nil if attr == :part && @pubid.respond_to?(:subpart=)
+        refresh_content!
+      end
+
+      def refresh_content!
+        store_content(@pubid.to_s) if @pubid
+      end
+    end
+  end
+end

data/lib/relaton/ogc/doctype.rb

@@ -0,0 +1,12 @@
+module Relaton
+  module Ogc
+    class Doctype < Bib::Doctype
+      attribute :content, :string, values: %w[
+        abstract-specification-topic best-practice change-request-supporting-document
+        community-practice community-standard discussion-paper engineering-report
+        other policy reference-model release-notes standard user-guide white-paper
+        test-suite draft-standard
+      ]
+    end
+  end
+end

data/lib/relaton/ogc/ext.rb

@@ -0,0 +1,15 @@
+require_relative "doctype"
+
+module Relaton
+  module Ogc
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :subdoctype, :string, values: %w[
+        conceptual-model conceptual-model-and-encoding conceptual-model-and-implementation
+        encoding extension implementation profile profile-with-extension general
+      ]
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-ogc"]
+    end
+  end
+end

data/lib/relaton/ogc/hit.rb

@@ -0,0 +1,18 @@
+module Relaton
+  module Ogc
+    class Hit < Core::Hit
+      # @return [Relaton::Ogc::ItemData]
+      def item
+        @item ||= begin
+          url = "#{HitCollection::ENDPOINT}#{hit[:file]}"
+          resp = Faraday.get(url) { |req| req.options.timeout = 10 }
+          return unless resp.status == 200
+
+          item = Item.from_yaml resp.body
+          item.fetched = Date.today.to_s
+          item
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ogc/hit_collection.rb

@@ -0,0 +1,111 @@
+require "faraday"
+require_relative "hit"
+
+module Relaton
+  module Ogc
+    class HitCollection < Core::HitCollection
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-ogc/v2/".freeze
+
+      # @return [self]
+      def find
+        return self if ref.nil? || ref.empty?
+
+        row = best_match ref
+        return self unless row
+
+        url = "#{ENDPOINT}#{row[:file]}"
+        resp = Faraday.get(url) { |req| req.options.timeout = 10 }
+        return self unless resp.status == 200
+
+        item = Item.from_yaml resp.body
+        item.fetched = Date.today.to_s
+        hit = Hit.new({ code: item.docidentifier[0]&.content, file: row[:file] }, self)
+        hit.instance_variable_set(:@item, item)
+        @array = [hit]
+        self
+      end
+
+      # @return [Relaton::Index::Type]
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :ogc, url: "#{ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+          pubid_class: ::Pubid::Ogc::Identifier
+        )
+      end
+
+      private
+
+      #
+      # Find the index row for a reference, latest revision first.
+      #
+      # Passing the pubid to `Index::Type#search` is what enables the binary
+      # search on `id.root.number` — with the plain string this used to pass,
+      # the whole index is scanned however the index was built. Selection is
+      # `Type#search`'s default with no block — pubid's asymmetric subset match
+      # (`Pubid::SubsetMatch`): the reference on the left, the row on the right,
+      # and a component it omits matches any
+      # value. `revision` is OGC's only optional component, so a
+      # bare `OGC 12-128` still finds the `r19` row, while `OGC 12-128r19`
+      # reaches only its own. `year` is always stated — the bsearch key is the
+      # `<nnn>` field alone, so one bucket holds every year that reused the
+      # number (`05-007` and `17-007` share bucket `007`).
+      #
+      # @param text [String]
+      # @return [Hash, nil]
+      #
+      # The substring-scan fallback this used to take when parsing failed is
+      # gone with the rescue. It was the same conflation the convention
+      # rejects: an unparseable string matched a SUBSTRING of the rendered
+      # ids, so an ambiguous reference silently resolved to whichever row
+      # sorted first, instead of telling the caller the reference is not an
+      # identifier. ecma, w3c and xsf never had one.
+      def best_match(text)
+        pubid = parse_ref text
+        rows = index.search(pubid)
+        rows.max_by { |r| [revision_key(r[:id].revision), r[:file]] }
+      end
+
+      #
+      # Order key for an OGC revision suffix.
+      #
+      # Revisions are `r<n>` with optional letter suffixes (`r3a`, `r12a`) plus
+      # the odd `a` and `c1`, so the number must be compared as an integer:
+      # `r2` beats `r14` as text but is the older document. The token itself
+      # breaks the tie, so `r3a` sorts above `r3` and a repeated lookup returns
+      # the same row (the index sort is not stable). An absent revision sorts
+      # below every present one.
+      #
+      # Highest revision is the newest published document in 105 of the 106
+      # multi-revision documents in the index, scored against each document's
+      # own `date[0].at`; the previous `min_by` on the rendered id matched 1.
+      #
+      # @param revision [String, nil]
+      # @return [Array]
+      #
+      def revision_key(revision)
+        [revision.to_s[/\d+/].to_i, revision.to_s]
+      end
+
+      #
+      # Parse a user reference into a `Pubid::Ogc::Identifier`, or nil.
+      #
+      # `Pubid::Ogc` takes the `OGC ` publisher token as optional and
+      # lowercases the revision, so `OGC 19-025r1`, `19-025r1`, `11-038R2` and
+      # the revision-less `16-079` all parse without normalization here.
+      #
+      # @param text [String]
+      # @return [Pubid::Ogc::Identifier, nil]
+      #
+      # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
+      # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
+      # `"..." is not a recognized standards identifier`
+      # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`), and `Db#fetch`
+      # logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
+      # Rescuing here would collapse "this identifier is malformed" into "no
+      # such document", leaving a caller unable to tell them apart.
+      def parse_ref(text)
+        ::Pubid::Ogc::Identifier.parse text.to_s.strip
+      end
+    end
+  end
+end

data/lib/relaton/ogc/item.rb

@@ -0,0 +1,13 @@
+require_relative "docidentifier"
+require_relative "ext"
+
+module Relaton
+  module Ogc
+    class Item < Iso::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/ogc/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Ogc
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/ogc/processor.rb

@@ -0,0 +1,59 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Ogc
+    class Processor < Core::Processor
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_ogc
+        @prefix = "OGC"
+        @defaultprefix = %r{^OGC\s}
+        @idtype = "OGC"
+        @datasets = %w[ogc-naming-authority]
+        @pubid_flavor = :Ogc
+      end
+
+      # @param code [String]
+      # @param date [String, nil]
+      # @param opts [Hash]
+      # @return [Relaton::Ogc::ItemData]
+      def get(code, date, opts)
+        require_relative "../ogc"
+        Bibliography.get(code, date, opts)
+      end
+
+      # @param source [String]
+      # @param opts [Hash]
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Ogc::ItemData]
+      def from_xml(xml)
+        require_relative "../ogc"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Ogc::ItemData]
+      def from_yaml(yaml)
+        require_relative "../ogc"
+        Item.from_yaml yaml
+      end
+
+      # @return [String]
+      def grammar_hash
+        require_relative "../ogc"
+        @grammar_hash ||= Relaton::Ogc.grammar_hash
+      end
+
+      def remove_index_file
+        require_relative "../ogc"
+        Relaton::Index.find_or_create(
+          :ogc, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end