relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/ecma/standard_parser.rb

@@ -0,0 +1,134 @@
+module Relaton
+  module Ecma
+    class StandardParser
+      include ParserCommon
+
+      ATTRS = %i[docidentifier title date source abstract relation edition ext].freeze
+
+      # @param [Nokogiri::XML::Element] hit document hit
+      # @param [Mechanize::Page] doc fetched document page
+      # @param [Hash] errors error tracking hash
+      def initialize(hit:, doc:, errors: {})
+        @hit = hit
+        @doc = doc
+        @errors = errors
+      end
+
+      # @return [Hash] bibliographic item attributes
+      def to_bib_hash
+        bib = default_bib_hash
+        ATTRS.each { |a| bib[a] = send "fetch_#{a}" }
+        bib
+      end
+
+      # @return [Array] precomputed translation sources
+      def translation_source
+        @translation_source ||= parse_translation_source
+      end
+
+      # @return [Array<Relaton::Ecma::Docidentifier>]
+      def fetch_docidentifier
+        result = super(@hit.text)
+        @errors[:standard_docidentifier] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Title>]
+      def fetch_title
+        result = @doc.xpath('//p[@class="ecma-item-short-description"]').map do |t|
+          Bib::Title.new(content: t.text.strip, language: "en", script: "Latn")
+        end
+        @errors[:standard_title] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::LocalizedMarkedUpString>]
+      def fetch_abstract
+        content = @doc.xpath('//div[@class="ecma-item-content"]/p').map do |a|
+          a.text.strip.squeeze(" ").gsub("\r\n", "")
+        end.join "\n"
+        return [] if content.empty?
+
+        result = [Bib::Abstract.new(content: content, language: "en", script: "Latn")]
+        @errors[:standard_abstract] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Date>]
+      def fetch_date
+        result = @doc.xpath('//p[@class="ecma-item-edition"]').map do |d|
+          date = d.text.split(", ").last
+          Bib::Date.new type: "published", at: date
+        end
+        @errors[:standard_date] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Uri>]
+      def fetch_source # rubocop:disable Metrics/AbcSize
+        source = []
+        source << Bib::Uri.new(type: "src", content: @hit[:href]) if @hit[:href]
+        ref = @doc.at('//div[@class="ecma-item-content-wrapper"]/span/a',
+                      '//div[@class="ecma-item-content-wrapper"]/a')
+        source << Bib::Uri.new(type: "pdf", content: ref[:href]) if ref
+        result = source + edition_translation_source(fetch_edition_content)
+        @errors[:standard_source] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Relation>]
+      def fetch_relation # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
+        edition_parser = EditionParser.new(doc: @doc, bib: {}, errors: @errors)
+        result = @doc.xpath("//ul[@class='ecma-item-archives']/li").filter_map do |rel|
+          ref, ed, date, vol = edition_parser.edition_id_parts rel.at("span").text
+          next if ed.nil? || ed.empty?
+
+          docid = Docidentifier.new(type: "ECMA", content: ref, primary: true)
+          source = rel.xpath("span/a").map { |l| Bib::Uri.new type: "pdf", content: l[:href] }
+          edition = Bib::Edition.new content: ed
+          extent = edition_parser.create_extent(vol)
+          @errors[:standard_relation_extent] &&= extent.nil?
+          bibitem = ItemData.new(
+            docidentifier: [docid], formattedref: Bib::Formattedref.new(content: ref), date: date, edition: edition,
+            source: source, extent: extent
+          )
+          Bib::Relation.new(type: "updates", bibitem: bibitem)
+        end
+        @errors[:standard_relation] &&= result.empty?
+        result
+      end
+
+      # @return [Relaton::Bib::Edition, nil]
+      def fetch_edition
+        cnt = fetch_edition_content
+        result = Bib::Edition.new(content: cnt) if cnt && !cnt.empty?
+        @errors[:standard_edition] &&= result.nil?
+        result
+      end
+
+      private
+
+      def fetch_edition_content
+        @doc.at('//p[@class="ecma-item-edition"]')&.text&.match(/^\d+(?=(?:st|nd|th|rd))/)&.to_s
+      end
+
+      def edition_translation_source(edition)
+        translation_source.select { |s| s[:ed] == edition }.map { |s| s[:source] }
+      end
+
+      def parse_translation_source
+        return [] unless @doc
+
+        @doc.xpath("//h2[.='Translations']/following-sibling::ul/li").map do |l|
+          a = l.at("span/a")
+          id = l.at("span").text
+          %r{\w+[\d-]+,\s(?<lang>\w+)\sversion,\s(?<ed>[\d.]+)(?:st|nd|rd|th)\sedition} =~ id
+          case lang
+          when "Japanese"
+            { ed: ed, source: Bib::Uri.new(type: "pdf", language: "ja", script: "Jpan", content: a[:href]) }
+          end
+        end.compact
+      end
+    end
+  end
+end

data/lib/relaton/ecma/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ecma
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-ecma".freeze
+    end
+  end
+end

data/lib/relaton/ecma.rb

@@ -0,0 +1,33 @@
+# Not lazy: DataFetcher names ::Pubid::Ecma::Identifier as the index
+# `pubid_class:`, and Docidentifier parses every docid through it.
+# (The IANA/IHO/IALA/OGC form; spec/relaton/lazy_loading_spec.rb guards that
+# this file is not itself loaded when a Db is built.)
+require "pubid"
+require "relaton/index"
+require "relaton/bib"
+require_relative "version"
+require_relative "ecma/util"
+require_relative "ecma/item_data"
+require_relative "ecma/item"
+require_relative "ecma/bibitem"
+require_relative "ecma/bibdata"
+require_relative "ecma/bibliography"
+
+module Relaton
+  module Ecma
+    # The one index this flavor builds and reads: pubid-keyed rows
+    # (`_type: pubid:ecma:*`), via `pubid_class: ::Pubid::Ecma::Identifier`.
+    # `relaton-data-ecma`'s crawler derives the legacy `index-v1` from these
+    # rows for released consumers, so it is not produced or read here.
+    INDEXFILE = "index-v2".freeze
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/etsi/bibdata.rb

@@ -0,0 +1,10 @@
+require_relative "item"
+
+module Relaton
+  module Etsi
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/etsi/bibitem.rb

@@ -0,0 +1,10 @@
+require_relative "item"
+
+module Relaton
+  module Etsi
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/etsi/bibliography.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Etsi
+    # Methods for search IANA standards.
+    module Bibliography
+      SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-etsi/refs/heads/v2/"
+
+      # @param text [String]
+      # @return [Relaton::Etsi::ItemData, nil]
+      def search(text) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        # An unrecognized reference raises Pubid::Errors::ParseError; like
+        # ISO we let it propagate — the CLI turns it into a friendly message
+        # and API callers rescue it themselves. Valid partial refs parse with
+        # the omitted refinements (version/date/part) left blank.
+        pubid = ::Pubid::Etsi.parse text
+
+        index = Relaton::Index.find_or_create :etsi, url: "#{SOURCE}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+                                                     pubid_class: ::Pubid::Etsi::Identifier
+        row = best_match(index, pubid)
+        return unless row
+
+        url = "#{SOURCE}#{row[:file]}"
+        resp = Net::HTTP.get_response URI(url)
+        return unless resp.code == "200"
+
+        Item.from_yaml(resp.body).tap { |item| item.fetched = Date.today.to_s }
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, Errno::ETIMEDOUT => e
+        raise Relaton::RequestError, e.message
+      end
+
+      # Match the reference against the index and return the most recent edition.
+      #
+      # The rows are selected with pubid's subset match `pubid === row`. A
+      # version or a date that the reference omits matches any value, so a
+      # bare `ETSI GS ZSM 012` matches every edition, while a fully-qualified
+      # ref matches only that edition. The class must be identical, so a base
+      # reference such as `ETSI ETR 310` does not reach its amendment or
+      # corrigendum (`ETR 310/C1`).
+      # The pubid — not a String — is passed to `index.search` so the index
+      # narrows candidates by number via binary search; each row's `:id` is
+      # already a Pubid::Etsi identifier (deserialized via `pubid_class`).
+      # `max_by` on `edition_key` picks the latest edition among the matches.
+      #
+      # pubid declares `parts` strict for ETSI, so a part-less reference asks
+      # for every part by matching each row **without its parts** (see
+      # #comparable). `#to_all_parts` cannot serve here: a
+      # `Pubid::AllPartsIdentifier` compares the document alone, so it also
+      # drops the version and the date, and
+      # `ETSI GR ZSM 011 V1.1.1 (2023-02)` would answer with V2.1.1.
+      #
+      # @param index [Relaton::Index::Type]
+      # @param pubid [::Pubid::Etsi::Identifier]
+      # @return [Hash, nil] the winning index row (`{ id:, file: }`)
+      def best_match(index, pubid)
+        all_parts = pubid.code&.parts.to_a.empty?
+        index.search(pubid) { |row| pubid === comparable(row[:id], all_parts) }
+             .max_by { |row| edition_key(row[:id]) }
+      end
+
+      # The row as the reference sees it: without its parts when the reference
+      # names none, unchanged otherwise. `#exclude` returns a copy, so the
+      # cached index id is untouched.
+      #
+      # @param id [::Pubid::Etsi::Identifier]
+      # @param all_parts [Boolean] the reference names no part
+      # @return [::Pubid::Etsi::Identifier]
+      def comparable(id, all_parts)
+        all_parts ? id.exclude(:part, :subpart, :parts) : id
+      end
+
+      # Sort key for one edition: the version numbers, then the publication date.
+      #
+      # ETSI versions are not zero-padded, so a comparison of the rendered id
+      # orders `V9.0.0` above `V19.0.0` and `ed.9` above `ed.11`, and a bare
+      # reference then resolves to an old edition. `Pubid::Etsi::Identifier` is
+      # not `Comparable` and its `<=>` returns nil, so the key comes from the
+      # parsed components: `version.version` holds the bare numbers (`"19.0.0"`,
+      # or `"9"` for the `ed.9` form) and `date` renders as `yyyy-mm`. Both
+      # delegate to `base` on a corrigendum/amendment id, so every row shape
+      # keys the same way. A missing version or date gives `[]` / `""`, which
+      # sort below any real value — hence the `.to_s` outside each `&.` chain.
+      #
+      # @param id [::Pubid::Etsi::Identifier]
+      # @return [Array(Array<Integer>, String)]
+      def edition_key(id)
+        [(id.version&.version).to_s.split(".").map(&:to_i), id.date.to_s]
+      end
+
+      # @param ref [String] the ETSI standard Code to look up
+      # @param year [String, nil] year
+      # @param opts [Hash] options
+      # @return [Relaton::Etsi::ItemData, nil]
+      def get(ref, _year = nil, _opts = {})
+        Util.info "Fetching from Relaton repository ...", key: ref
+        result = search(ref)
+        unless result
+          Util.info "Not found.", key: ref
+          return
+        end
+
+        Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
+        result
+      end
+
+      extend Bibliography
+    end
+  end
+end

data/lib/relaton/etsi/data_fetcher.rb

@@ -0,0 +1,167 @@
+require "date"
+require "json"
+require "mechanize"
+require "relaton/core"
+require "relaton/index"
+require_relative "../etsi"
+require_relative "data_parser"
+
+module Relaton
+  module Etsi
+    class DataFetcher < Core::DataFetcher
+      PAGE_SIZE = 50
+
+      # `version=1` keeps the superseded EDITIONS of a document in the result
+      # set; `version=0` returns only the current edition of each branch. The
+      # status flags below are a separate axis and already permissive, so this
+      # is what makes e.g. all three editions of `ETSI EN 319 142-1` retrievable
+      # instead of one. It also makes the crawl about 2.4 times larger — see
+      # lib/relaton/etsi/CLAUDE.md before changing it back.
+      SOURCEURL = "https://www.etsi.org/custom/standardssearch/data.php?format=json&includeScope=1&" \
+        "page=%<page>s&search=&title=1&etsiNumber=1&content=1&version=1&onApproval=1&published=1&" \
+        "withdrawn=1&historical=1&isCurrent=1&superseded=1&startDate=1988-01-15&endDate=%<date>s&" \
+        "harmonized=0&keyword=&TB=&stdType=&frequency=&mandate=&collection=&sort=1&x=%<timestamp>s".freeze
+
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :etsi, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Etsi::Identifier
+        )
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      #
+      # Fetch all ETSI documents from the ETSI website.
+      #
+      # @param [Object] _source unused, required by superclass interface
+      #
+      def fetch(_source = nil)
+        first_page = fetch_page(1)
+        process_records(first_page)
+        fetch_remaining_pages(first_page)
+        index.save
+        report_errors
+      end
+
+      def fetch_remaining_pages(first_page)
+        total = first_page.first ? first_page.first["total_count"].to_i : 0
+        total_pages = (total / PAGE_SIZE.to_f).ceil
+        (2..total_pages).each do |page|
+          records = fetch_page(page)
+          break if records.empty?
+
+          process_records(records)
+        end
+      end
+
+      def fetch_page(page)
+        date = Time.now.to_date + 1
+        timestamp = (Time.now.to_f * 1000).to_i
+        url = format(SOURCEURL, page: page, date: date, timestamp: timestamp)
+        JSON.parse(fetch_with_retry(url))
+      end
+
+      def process_records(records)
+        records.each do |record|
+          save DataParser.new(normalize(record), @errors).parse
+        end
+      end
+
+      def normalize(record)
+        {
+          "ETSI deliverable" => record["ETSI_DELIVERABLE"],
+          "title" => record["TITLE"],
+          "Details link" => "https://webapp.etsi.org/workprogram/Report_WorkItem.asp?WKI_ID=#{record['wki_id']}",
+          "PDF link" => "https://www.etsi.org/deliver/#{record['EDSpathname']}#{record['EDSPDFfilename']}",
+          "Status" => derive_status(record),
+          "Keywords" => record["Keywords"].to_s,
+          "Technical body" => record["TB"],
+          "Scope" => record["Scope"],
+        }
+      end
+
+      def derive_status(record)
+        return "Withdrawn" if record["ACTION_TYPE"] == "WD"
+
+        code = record["STATUS_CODE"].to_i
+        return "On Approval" if code < 12
+        return "Historical" if code == 13
+
+        "Published"
+      end
+
+      NETWORK_ERRORS = [
+        Mechanize::Error, Net::OpenTimeout, Net::ReadTimeout,
+        SocketError, Errno::ECONNRESET
+      ].freeze
+
+      def fetch_with_retry(url, retries: 3, delay: 2) # rubocop:disable Metrics/MethodLength
+        attempt = 0
+        begin
+          Mechanize.new.get(url).body
+        rescue *NETWORK_ERRORS => e
+          attempt += 1
+          if attempt <= retries
+            Util.info "Fetch failed (#{e.message}), " \
+                      "retrying (#{attempt}/#{retries})..."
+            sleep delay * attempt
+            retry
+          end
+          raise
+        end
+      end
+
+      def save(bib)
+        id = bib.docidentifier.first.content
+        pid = pubid(id) or return # skip ids pubid can't parse/serialize
+
+        # Distinct docids can sanitize to one filename (`output_file` collapses
+        # `/`, `-`, `.`, `:` and `()` alike, and ETSI ids use all of them), which
+        # used to overwrite silently. Take a path of our own. There is no @files
+        # here, so a repeat of the SAME id still resolves to one path and
+        # overwrites itself, as before.
+        file = unique_output_file id
+        if file != output_file(id)
+          Util.warn "File #{output_file id} already exists. Docid: #{id}. Writing #{file} instead."
+        end
+        File.write file, serialize(bib), encoding: "UTF-8"
+        index.add_or_update pid, file
+      end
+
+      #
+      # Parse an ETSI docid into a Pubid::Etsi identifier, or nil when it can't
+      # be parsed or serialized. Storing the pubid object (not its hash) lets
+      # Relaton::Index sort the index by id number and serialize each id to its
+      # `_type: pubid:etsi:*` hash on save. A single unindexable id must never
+      # abort the crawl or corrupt the index, so it is skipped defensively
+      # (mirrors Relaton::Nist#pubid / Relaton::Jcgm#add_to_index). ETSI's whole
+      # published corpus parses on the pinned pubid; the guard protects against a
+      # future malformed record.
+      #
+      # @param [String] id docidentifier content, e.g. "ETSI GS ZSM 012 V1.1.1 (2022-12)"
+      # @return [::Pubid::Etsi::Identifier, nil]
+      #
+      def pubid(id)
+        pid = ::Pubid::Etsi.parse id
+        pid.to_hash # skip ids whose to_hash raises so index.save can't fail
+        pid
+      rescue StandardError
+        nil
+      end
+
+      def to_yaml(bib)
+        bib.to_yaml
+      end
+
+      def to_xml(bib)
+        bib.to_xml bibdata: true
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+    end
+  end
+end

data/lib/relaton/etsi/data_parser.rb

@@ -0,0 +1,208 @@
+require_relative "pubid"
+module Relaton
+  module Etsi
+    class DataParser
+      ATTRS = %i[
+        title docnumber source date docidentifier version status contributor
+        keyword ext abstract language script
+      ].freeze
+
+      def initialize(row, errors = {})
+        @row = row
+        @errors = errors
+      end
+
+      def parse
+        args = ATTRS.to_h { |attr| [attr, send(attr)] }
+        ItemData.new(type: "standard", **args)
+      end
+
+      def pubid
+        return @pubid if defined?(@pubid)
+
+        unless @row["ETSI deliverable"]
+          @errors[:pubid] &&= true
+          @pubid = nil
+          return @pubid
+        end
+
+        @errors[:pubid] &&= false
+        @pubid = PubId.parse(@row["ETSI deliverable"])
+      end
+
+      def title
+        result = if @row["title"]
+                   [Bib::Title.new(
+                     content: @row["title"], language: "en", script: "Latn",
+                   )]
+                 else
+                   []
+                 end
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      def docnumber
+        @row["ETSI deliverable"]
+      end
+
+      def source
+        urls = []
+        if @row["Details link"]
+          urls << Bib::Uri.new(content: @row["Details link"], type: "src")
+        end
+        if @row["PDF link"]
+          urls << Bib::Uri.new(content: @row["PDF link"], type: "pdf")
+        end
+        @errors[:source] &&= urls.empty?
+        urls
+      end
+
+      def date
+        result = if pubid&.date
+                   [Bib::Date.new(type: "published", at: pubid.date)]
+                 else
+                   []
+                 end
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      def docidentifier
+        result = if @row["ETSI deliverable"]
+                   [Bib::Docidentifier.new(
+                     content: @row["ETSI deliverable"],
+                     type: "ETSI",
+                     primary: true,
+                   )]
+                 else
+                   []
+                 end
+        @errors[:docidentifier] &&= result.empty?
+        result
+      end
+
+      def version
+        return [] unless pubid&.version
+
+        [Bib::Version.new(draft: pubid.version)]
+      end
+
+      def status
+        unless @row["Status"]
+          @errors[:status] &&= true
+          return
+        end
+
+        stage = @row["Status"]
+        if stage == "On Approval"
+          stage = "#{pubid&.type} approval"
+        end
+        @errors[:status] &&= false
+        Status.new(stage: stage)
+      end
+
+      def contributor
+        contribs = [publisher_contributor]
+        contribs << committee_contributor if @row["Technical body"]
+        contribs
+      end
+
+      def publisher_contributor
+        org = Bib::Organization.new(
+          name: [etsi_org_name], abbreviation: etsi_abbreviation,
+        )
+        role = Bib::Contributor::Role.new(type: "publisher")
+        Bib::Contributor.new(organization: org, role: [role])
+      end
+
+      def etsi_org_name
+        Bib::TypedLocalizedString.new(
+          content: "European Telecommunications Standards Institute",
+          language: "en",
+          script: "Latn",
+        )
+      end
+
+      def etsi_abbreviation
+        Bib::LocalizedString.new(
+          content: "ETSI",
+          language: "en",
+          script: "Latn",
+        )
+      end
+
+      def committee_contributor
+        desc = Bib::LocalizedMarkedUpString.new(content: "committee")
+        role = Bib::Contributor::Role.new(
+          type: "author", description: [desc],
+        )
+        Bib::Contributor.new(organization: committee_org, role: [role])
+      end
+
+      def committee_org
+        Bib::Organization.new(
+          name: [etsi_name],
+          abbreviation: etsi_abbreviation,
+          subdivision: [technical_committee_subdivision],
+        )
+      end
+
+      def etsi_name
+        Bib::TypedLocalizedString.new(
+          content: "European Telecommunications Standards Institute",
+        )
+      end
+
+      def technical_committee_subdivision
+        Bib::Subdivision.new(
+          name: [Bib::TypedLocalizedString.new(
+            content: @row["Technical body"],
+          )],
+          type: "technical-committee",
+        )
+      end
+
+      def keyword
+        unless @row["Keywords"]
+          @errors[:keyword] &&= true
+          return []
+        end
+
+        result = @row["Keywords"].split(",").map do |kw|
+          Bib::Keyword.new(vocab: Bib::LocalizedString.new(content: kw.strip, language: "en", script: "Latn"))
+        end
+        @errors[:keyword] &&= result.empty?
+        result
+      end
+
+      def ext
+        Ext.new(doctype: doctype, flavor: "etsi")
+      end
+
+      def doctype
+        return unless pubid
+
+        Doctype.create_from_abbreviation pubid.type
+      end
+
+      def abstract
+        result = if @row["Scope"]
+                   [Bib::Abstract.new(content: @row["Scope"], language: "en", script: "Latn")]
+                 else
+                   []
+                 end
+        @errors[:abstract] &&= result.empty?
+        result
+      end
+
+      def language
+        ["en"]
+      end
+
+      def script
+        ["Latn"]
+      end
+    end
+  end
+end