relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/ecma/data_fetcher.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require "English"
+require "mechanize"
+require "relaton/core"
+require_relative "../ecma"
+require_relative "parser_common"
+require_relative "page_fetcher"
+require_relative "standard_parser"
+require_relative "memento_parser"
+require_relative "edition_parser"
+require_relative "data_parser"
+
+module Relaton
+  module Ecma
+    class DataFetcher < Core::DataFetcher
+      URL = "https://www.ecma-international.org/publications-and-standards/"
+      SOURCES = %w[standards technical-reports mementos].freeze
+
+      def index
+        # `pubid_class:` on the producer too: FileIO#save only calls `to_hash`
+        # for instances of it, so without it the crawl writes v1-shaped rows
+        # under a v2 name, silently.
+        @index ||= Relaton::Index.find_or_create(
+          :ecma, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Ecma::Identifier
+        )
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def agent
+        @agent ||= Mechanize.new.tap { |a| a.user_agent_alias = Mechanize::AGENT_ALIASES.keys.sample }
+      end
+
+      # @param bib [Relaton::Ecma::ItemData]
+      def write_file(bib)
+        # Two distinct ids can sanitize to one filename, in which case the
+        # second document used to be dropped outright. Give it a path of its
+        # own; a genuine duplicate (same id) still resolves to one path and is
+        # still skipped.
+        file = unique_output_file filename_id(bib)
+        # A reserved path only ever belongs to one id, so a hit here is the same
+        # document again. Checked FIRST: a disambiguated path stays != filename
+        # forever, so gating this on that comparison would make a repeat of a
+        # disambiguated id skip the skip and overwrite its own file.
+        if @files.include? file
+          Util.warn "Duplicate file #{file}"
+          return
+        end
+
+        Util.warn "Duplicate file #{filename bib}; writing #{file} instead" if file != filename(bib)
+        @files << file
+        File.write file, serialize(bib), encoding: "UTF-8"
+        add_to_index bib, file
+      end
+
+      #
+      # Index the document, or record why it could not be indexed.
+      #
+      # An id pubid cannot rebuild is recorded in `@errors` — the inherited
+      # `report_errors` logs a String value as the message, and its GhIssue
+      # channel opens a GitHub issue at the end of the crawl — and the row is
+      # skipped rather than indexed unparsed: `Relaton::Index` rejects the WHOLE
+      # index if a single row fails to deserialize, and its sort calls
+      # `.root.number` on every id. The data file is already written by the
+      # caller, so the document is unindexed, never lost. (The 3GPP/W3C shape.)
+      #
+      # @param bib [Relaton::Ecma::ItemData]
+      # @param file [String] path the document was written to
+      #
+      def add_to_index(bib, file)
+        id = index_id bib
+        return index.add_or_update(id, file) if id
+
+        docid = bib.docidentifier[0]&.content || file
+        @errors[docid.to_s] = "Unparseable primary id `#{docid}` was not indexed (#{file})"
+      end
+
+      def filename(bib)
+        output_file filename_id(bib)
+      end
+
+      # The docid the filename is derived from.
+      def filename_id(bib)
+        id = bib.docidentifier[0].content
+        id += " #{bib.edition.content}" if bib.edition
+        locality = locality_with_volume bib
+        id += " #{locality.reference_from}" if locality
+        id
+      end
+
+      #
+      # The index key: a `Pubid::Ecma::Identifier` carrying the number, the
+      # edition and the volume.
+      #
+      # Built from the MODEL, never from a rendered string — the same three
+      # fields `#filename_id` reads — so the crawl cannot lose a component to a
+      # parse or to a render default. The base identifier is the docidentifier's
+      # own pubid, **duplicated** first: `edition` and `volume` are index
+      # metadata, and every `ECMA-269` volume file carries the bare
+      # `docidentifier: ECMA-269`, so setting them on the shared object would
+      # promote the document's own printed id to the index form.
+      #
+      # @param bib [Relaton::Ecma::ItemData]
+      # @return [Pubid::Ecma::Identifier, nil] nil if pubid rejects the docid
+      #
+      def index_id(bib)
+        pubid = bib.docidentifier[0]&.pubid&.dup
+        return unless pubid
+
+        pubid.edition = bib.edition.content if bib.edition
+        locality = locality_with_volume bib
+        pubid.volume = locality.reference_from if locality
+        pubid
+      end
+
+      def locality_with_volume(bib)
+        bib.extent.each do |e|
+          locality = e.locality.find { |l| l.type == "volume" }
+          return locality if locality
+        end
+        nil
+      end
+
+      def to_xml(bib) = bib.to_xml(bibdata: true)
+      def to_yaml(bib) = bib.to_yaml
+      def to_bibxml(bib) = bib.to_rfcxml
+
+      # @param hit [Nokogiri::XML::Element]
+      def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        DataParser.new(hit, @errors).parse.each { |item| write_file item }
+      end
+
+      # @param type [String]
+      def html_index(type) # rubocop:disable Metrics/MethodLength
+        result = agent.get "#{URL}#{type}/"
+        # @last_call_time = Time.now
+        result.xpath(
+          "//li/span[1]/a",
+          "//div[contains(@class, 'entry-content-wrapper')][.//a[.='Download']]",
+        ).each do |hit|
+          parse_page(hit)
+        rescue StandardError => e
+          Util.error { "#{e.message}\n#{e.backtrace}" }
+        end
+      end
+
+      #
+      # Fetch data from Ecma website.
+      #
+      # @return [void]
+      #
+      def fetch(_ = nil)
+        SOURCES.each { |source| html_index source }
+        index.save
+        report_errors
+      end
+    end
+  end
+end

data/lib/relaton/ecma/data_parser.rb

@@ -0,0 +1,49 @@
+module Relaton
+  module Ecma
+    class DataParser
+      include ParserCommon
+
+      #
+      # Initialize parser
+      #
+      # @param [Nokogiri::XML::Element] hit document hit
+      # @param [Hash] errors error tracking hash
+      #
+      def initialize(hit, errors = {})
+        @hit = hit
+        @errors = errors
+      end
+
+      # @return [Array<Relaton::Ecma::ItemData>]
+      def parse
+        if @hit[:href]
+          parse_standard
+        else
+          parse_memento
+        end
+      end
+
+      private
+
+      def parse_standard
+        doc = PageFetcher.new.get(@hit[:href])
+        parser = StandardParser.new(hit: @hit, doc: doc, errors: @errors)
+        bib = parser.to_bib_hash
+        bib[:contributor] = contributor
+        items = [ItemData.new(**bib)]
+        edition_parser = EditionParser.new(
+          doc: doc, bib: bib, errors: @errors,
+          translation_source: parser.translation_source
+        )
+        items + edition_parser.parse
+      end
+
+      def parse_memento
+        parser = MementoParser.new(hit: @hit, errors: @errors)
+        bib = parser.to_bib_hash
+        bib[:contributor] = contributor
+        [ItemData.new(**bib)]
+      end
+    end
+  end
+end

data/lib/relaton/ecma/docidentifier.rb

@@ -0,0 +1,124 @@
+module Relaton
+  module Ecma
+    # Structured ECMA document identifier. The docid string is the BARE printed
+    # form ("ECMA-269", "ECMA TR/101", "ECMA MEM/2021"), and it is parsed into a
+    # `Pubid::Ecma::Identifier` kept in `@pubid` while the lutaml `content`
+    # attribute stays a plain string for serialization.
+    #
+    # Parsing is **soft**: `content=` lazily requires pubid and rescues
+    # `LoadError`/`StandardError`, so a missing pubid gem or non-ECMA content
+    # leaves `@pubid` nil rather than raising.
+    #
+    # ## Why this class exists, and the one rule that is ECMA-specific
+    #
+    # `Pubid::Ecma::Identifier#to_s` renders the edition and the volume **by
+    # default** — `"ECMA-269 ed3 vol2"`. That default is deliberate on the pubid
+    # side: `Relaton::Index::Type#add_or_update` keys on a bare `id.to_s`, and
+    # 740 of the 804 published rows carry an edition, so without it 383 rows
+    # collapse onto another row's key and vanish from the index.
+    #
+    # A **document's own** docidentifier is the opposite. Every `ECMA-269`
+    # volume file in `relaton-data-ecma` carries `docidentifier: ECMA-269` and
+    # the same title — the edition and the volume are index metadata, not part
+    # of the printed id. So `refresh_content!` must opt OUT of both, or a
+    # mutation would silently promote the stored content to the index form.
+    #
+    # This is the mirror image of `Relaton::ThreeGpp::Docidentifier`, whose
+    # `refresh_content!` must pass `with_publisher: true` because ITS pubid
+    # defaults to the index rendering and its stored content is the fuller one.
+    # The two look contradictory only until you notice each one re-renders what
+    # its own stored `content` already holds.
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      # Capture the inherited (LocalizedMarkedUpString) content setter before
+      # overriding #content=, so #refresh_content! can write the re-rendered
+      # string back WITHOUT re-parsing (a re-parse would rebuild @pubid from the
+      # string and discard in-place mutations).
+      alias_method :store_content, :content=
+
+      def initialize(attrs = {}, options = {})
+        pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
+        attrs[:content] ||= render(pubid) if pubid
+        super
+        @pubid = pubid if pubid
+      end
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          require "pubid"
+          ::Pubid::Ecma::Identifier.parse(value.to_s)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+
+      # `Pubid::Ecma::Identifier` models number/part/subpart/edition/volume, so
+      # the mapping is:
+      #
+      # - **`remove_date!` → clears `edition`.** ECMA has no date component;
+      #   `edition` is its version discriminator, so clearing it yields the
+      #   version-agnostic ("most recent") reference. It is invisible in the
+      #   rendered bare form, which never carried the edition to begin with —
+      #   but the identifier really does change, and a consumer reading
+      #   `#pubid` sees it.
+      # - **`remove_part!` → clears `part`/`subpart`.** Real here:
+      #   "ECMA-418-1" -> "ECMA-418".
+      # - **`to_all_parts!` → both, then wraps `@pubid` in pubid's `AllParts`.**
+      #   `content` stays the plain stripped id — ECMA's own renderer has no
+      #   "(all parts)" marker, and `content` is cached from the pre-wrap
+      #   pubid on purpose. `#pubid` itself, read directly, now answers
+      #   `all_parts? == true` and renders WITH pubid's generic marker
+      #   (`#to_s`), since it's the wrapper.
+      #
+      # All three no-op safely when `@pubid` is nil, so `Bib::ItemData`'s
+      # `#to_all_parts` / `#to_most_recent_reference` never raise on ECMA items.
+
+      def remove_part!
+        clear_attrs! :part
+      end
+
+      def remove_date!
+        clear_attrs! :edition
+      end
+
+      def to_all_parts!
+        return if !@pubid || @pubid.all_parts?
+
+        # `#exclude` (no args) rebuilds a full independent copy — `remove_part!`
+        # / `remove_date!` mutate `@pubid` in place, so a bare reference here
+        # would lose the original part/edition to that mutation too.
+        original = @pubid.exclude
+        remove_part!
+        remove_date!
+        @pubid = original.to_all_parts
+      end
+
+      private
+
+      def clear_attrs!(*attrs)
+        return unless @pubid
+
+        cleared = attrs.select { |attr| @pubid.respond_to?("#{attr}=") }
+        return if cleared.empty?
+
+        cleared.each { |attr| @pubid.public_send("#{attr}=", nil) }
+        @pubid.subpart = nil if attrs.include?(:part) && @pubid.respond_to?(:subpart=)
+        refresh_content!
+      end
+
+      def refresh_content!
+        store_content(render(@pubid)) if @pubid
+      end
+
+      # The bare document form. See the class comment for why both flags are
+      # opted out of, and why 3GPP's twin does the opposite.
+      def render(pubid)
+        pubid.to_s(with_edition: false, with_volume: false)
+      end
+    end
+  end
+end

data/lib/relaton/ecma/edition_parser.rb

@@ -0,0 +1,80 @@
+module Relaton
+  module Ecma
+    class EditionParser
+      include ParserCommon
+
+      # @param [Mechanize::Page] doc document page
+      # @param [Hash] bib base bibliographic item attributes
+      # @param [Hash] errors error tracking hash
+      # @param [Array] translation_source precomputed translation sources
+      def initialize(doc:, bib:, errors: {}, translation_source: [])
+        @doc = doc
+        @bib = bib
+        @errors = errors
+        @translation_source = translation_source
+      end
+
+      # @return [Array<Relaton::Ecma::ItemData>] editions
+      def parse # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        return [] unless @doc
+
+        docid = @bib[:docidentifier]
+        @doc.xpath('//div[@id="main"]/div[1]/div/main/article/div/div/standard/div/ul/li').map do |hit|
+          bib = @bib.dup
+          id, ed, bib[:date], vol = edition_id_parts hit.at("./span", "./a").text
+          bib[:source] = edition_source(hit) + edition_translation_source(ed)
+          next if ed.nil? || ed.empty?
+
+          bib[:docidentifier] = id.nil? || id.empty? ? docid : fetch_docidentifier(id)
+          @errors[:edition_docidentifier] &&= bib[:docidentifier].empty?
+          bib[:edition] = Bib::Edition.new(content: ed)
+          bib[:extent] = create_extent(vol)
+          @errors[:edition_extent] &&= bib[:extent].nil?
+          ItemData.new(**bib)
+        end.compact
+      end
+
+      #
+      # Parse edition and date
+      #
+      # @param [String] text identifier text
+      #
+      # @return [Array<String,nil,Array<Relaton::Bib::Date>>] edition and date
+      #
+      def edition_id_parts(text) # rubocop:disable Metrics/MethodLength
+        %r{^
+          (?<id>\w+(?:[\d-]+|\sTR/\d+)),?\s
+          (?:Volume\s(?<vol>[\d.]+),?\s)?
+          (?<ed>[\d.]+)(?:st|nd|rd|th)?\sedition
+          (?:[,.]\s(?<dt>\w+\s\d+))?
+        }x =~ text
+        date = [dt].compact.map do |d|
+          on = Date.strptime(d, "%B %Y").strftime("%Y-%m")
+          Bib::Date.new(type: "published", at: on)
+        end
+        [id, ed, date, vol]
+      end
+
+      def edition_source(hit)
+        es = { "src" => hit.at("./a"), "pdf" => hit.at("./span/a") }.map do |type, a|
+          Bib::Uri.new(type: type, content: a[:href]) if a
+        end.compact
+        @errors[:edition_source] &&= es.empty?
+        es
+      end
+
+      def create_extent(vol)
+        return unless vol && !vol.empty?
+
+        locality = Bib::Locality.new(type: "volume", reference_from: vol)
+        [Bib::Extent.new(locality: [locality])]
+      end
+
+      private
+
+      def edition_translation_source(edition)
+        @translation_source.select { |s| s[:ed] == edition }.map { |s| s[:source] }
+      end
+    end
+  end
+end

data/lib/relaton/ecma/ext.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Ecma
+    class Ext < Bib::Ext
+      def get_schema_version = Relaton.schema_versions["relaton-model-ecma"]
+    end
+  end
+end

data/lib/relaton/ecma/item.rb

@@ -0,0 +1,13 @@
+require_relative "docidentifier"
+require_relative "ext"
+
+module Relaton
+  module Ecma
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/ecma/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Ecma
+    class ItemData < Relaton::Bib::ItemData
+    end
+  end
+end

data/lib/relaton/ecma/memento_parser.rb

@@ -0,0 +1,60 @@
+module Relaton
+  module Ecma
+    class MementoParser
+      include ParserCommon
+
+      ATTRS = %i[docidentifier title date source ext].freeze
+
+      # @param [Nokogiri::XML::Element] hit document hit
+      # @param [Hash] errors error tracking hash
+      def initialize(hit:, errors: {})
+        @hit = hit
+        @errors = errors
+      end
+
+      # @return [Hash] bibliographic item attributes
+      def to_bib_hash
+        bib = default_bib_hash
+        ATTRS.each { |a| bib[a] = send "fetch_#{a}" }
+        bib
+      end
+
+      private
+
+      # @return [Array<Relaton::Ecma::Docidentifier>]
+      def fetch_docidentifier
+        code = "ECMA MEM/#{@hit.at('div[1]//p').text}"
+        docid = super(code)
+        @errors[:memento_docidentifier] &&= docid.empty?
+        docid
+      end
+
+      # @return [Array<Relaton::Bib::Title>]
+      def fetch_title
+        year = @hit.at("div[1]//p").text
+        content = "\"Memento #{year}\" for year #{year}"
+        result = [Bib::Title.new(content: content, language: "en", script: "Latn")]
+        @errors[:memento_title] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Date>]
+      def fetch_date
+        date = @hit.at("div[2]//p").text
+        on = Date.strptime(date, "%B %Y").strftime "%Y-%m"
+        result = [Bib::Date.new(type: "published", at: on)]
+        @errors[:memento_date] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Uri>]
+      def fetch_source
+        result = @hit.xpath("./div/section/div/p/a").map do |a|
+          Bib::Uri.new(type: "pdf", content: a[:href])
+        end
+        @errors[:memento_source] &&= result.empty?
+        result
+      end
+    end
+  end
+end

data/lib/relaton/ecma/page_fetcher.rb

@@ -0,0 +1,39 @@
+module Relaton
+  module Ecma
+    class PageFetcher
+      RETRIES = 3
+
+      def initialize
+        @agent = Mechanize.new
+        @agent.user_agent_alias = Mechanize::AGENT_ALIASES.keys[rand(21)]
+      end
+
+      #
+      # Get page with retries
+      #
+      # A failed fetch must raise, not fall out of the loop. `RETRIES.times`
+      # returns its receiver, so the exhausted loop used to hand the caller the
+      # Integer 3 as if it were a page, and `StandardParser#fetch_title`
+      # reported the transport error as `undefined method 'xpath' for an
+      # instance of Integer`. `DataFetcher#html_index` already rescues per hit,
+      # so one unreachable page is skipped with a readable message.
+      #
+      # @param [String] url url to fetch
+      #
+      # @return [Mechanize::Page] document
+      # @raise [Relaton::RequestError] after RETRIES attempts
+      #
+      def get(url)
+        error = nil
+        RETRIES.times do |n|
+          sleep n
+          return @agent.get url
+        rescue StandardError => e
+          error = e
+          Util.error e.message
+        end
+        raise Relaton::RequestError, "Could not access #{url}: #{error.message}"
+      end
+    end
+  end
+end

data/lib/relaton/ecma/parser_common.rb

@@ -0,0 +1,33 @@
+module Relaton
+  module Ecma
+    module ParserCommon
+      def default_bib_hash
+        {
+          type: "standard", language: ["en"], script: ["Latn"], place: [Bib::Place.new(city: "Geneva")]
+        }
+      end
+
+      def contributor
+        orgname = Bib::TypedLocalizedString.new(content: "Ecma International", language: "en", script: "Latn")
+        org = Bib::Organization.new name: [orgname]
+        role = Bib::Contributor::Role.new type: "publisher"
+        [Bib::Contributor.new(organization: org, role: [role])]
+      end
+
+      # @return [Array<Relaton::Ecma::Docidentifier>]
+      def fetch_docidentifier(id = nil)
+        return [] if id.nil? || id.empty?
+
+        [Docidentifier.new(type: "ECMA", content: id, primary: true)]
+      end
+
+      def fetch_ext
+        Ext.new(doctype: fetch_doctype, flavor: "ecma")
+      end
+
+      def fetch_doctype
+        Bib::Doctype.new content: "document"
+      end
+    end
+  end
+end

data/lib/relaton/ecma/processor.rb

@@ -0,0 +1,69 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Ecma
+    class Processor < Relaton::Core::Processor
+      def initialize
+        @short = :relaton_ecma
+        @prefix = "ECMA"
+        @defaultprefix = /^ECMA(-|\s)/
+        @idtype = "ECMA"
+        @datasets = %w[ecma-standards]
+        @pubid_flavor = :Ecma
+      end
+
+      # @param code [String]
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [Relaton::Ecma::ItemData, nil]
+      def get(code, date, opts)
+        require_relative "../ecma"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Fetch all the documents from a source
+      #
+      # @param [String] source source name (iec-harmonized-all, iec-harmonized-latest)
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format output format (xml, yaml, bibxml)
+      #
+      def fetch_data(source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(source, **opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Ecma::ItemData]
+      def from_xml(xml)
+        require_relative "../ecma"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Ecma::ItemData]
+      def from_yaml(yaml)
+        require_relative "../ecma" # defines Item — cache reads hit this cold
+        Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../ecma"
+        @grammar_hash ||= Ecma.grammar_hash
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../ecma"
+        Relaton::Index.find_or_create(
+          :ECMA, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end