relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/oasis/data_parser_utils.rb

@@ -0,0 +1,330 @@
+require "mechanize"
+require_relative "browser_agent"
+
+module Relaton
+  module Oasis
+    # Common methods for document and part parsers.
+    module DataParserUtils
+      RETRIABLE_PAGE_ERRORS = [
+        Errno::ETIMEDOUT,
+        Net::OpenTimeout,
+        Ferrum::TimeoutError,
+        Ferrum::PendingConnectionsError,
+        Ferrum::StatusError,
+      ].freeze
+      #
+      # Parse contributor.
+      #
+      # @return [Array<Bib::Contributor>] contributors
+      #
+      def parse_contributor
+        result = publisher_oasis + parse_authorizer +
+          parse_editorialgroup_contributor +
+          parse_chairs + parse_editors
+        @errors[:contributor] &&= result.empty?
+        result
+      end
+
+      def publisher_oasis
+        org = Bib::Organization.new(
+          name: [Bib::TypedLocalizedString.new(content: "OASIS")],
+          uri: [Bib::Uri.new(type: "uri", content: "https://www.oasis-open.org/")],
+        )
+        role = [
+          Bib::Contributor::Role.new(
+            type: "authorizer",
+            description: [Bib::LocalizedMarkedUpString.new(
+              content: "Standards Development Organization",
+            )],
+          ),
+          Bib::Contributor::Role.new(type: "publisher"),
+        ]
+        [Bib::Contributor.new(organization: org, role: role)]
+      end
+
+      def parse_editors_from_text # rubocop:disable Metrics/MethodLength
+        result = if text
+                   text.match(/(?<=Edited\sby\s)[^.]+/).to_s
+                     .split(/,?\sand\s|,\s/).map do |c|
+                     role = [Bib::Contributor::Role.new(type: "editor")]
+                     Bib::Contributor.new(role: role,
+                                          person: create_person(c))
+                   end
+                 else
+                   []
+                 end
+        @errors[:editors] &&= result.empty?
+        result
+      end
+
+      def page
+        return @page if defined? @page
+
+        @page = nil
+        return @page unless link_node && link_node[:href].match?(/\.html$/)
+
+        if @agent
+          doc = retry_page(link_node[:href], @agent)
+          @page = doc if doc && @agent.last_status == 200
+        else
+          # No injected agent (e.g. unit tests with VCR cassettes): fall back
+          # to a Mechanize request — VCR can intercept it.
+          agent = Mechanize.new
+          agent.agent.allowed_error_codes = [403, 404, 503]
+          resp = retry_page(link_node[:href], agent)
+          @page = resp if resp && resp.code == "200"
+        end
+      end
+
+      #
+      # Retry to get page.
+      #
+      # @param [String] url page URL
+      # @param [#get] agent HTTP client responding to #get(url)
+      # @param [Integer] retries number of retries
+      #
+      # @return [Nokogiri::HTML::Document, Mechanize::Page, nil] page or nil
+      #
+      def retry_page(url, agent, retries = 3)
+        sleep 1 # to avoid 429 error
+        agent.get url
+      rescue *RETRIABLE_PAGE_ERRORS => e
+        retry if (retries -= 1).positive?
+        Util.error "Failed to get page `#{url}`\n#{e.message}"
+        nil
+      end
+
+      def parse_chairs # rubocop:disable Metrics/MethodLength
+        result = if page
+                   xpath = "//p[preceding-sibling::p" \
+                           "[starts-with(., 'Chair')]]" \
+                           "[following-sibling::p" \
+                           "[starts-with(., 'Editor')]]"
+                   page.xpath(xpath).map do |p|
+                     create_contribution_info(p, "editor", ["Chair"])
+                   end.compact
+                 else
+                   []
+                 end
+        @errors[:chairs] &&= result.empty?
+        result
+      end
+
+      def parse_editors # rubocop:disable Metrics/MethodLength
+        result = if page
+                   xpath = "//p[contains(@class, 'Contributor')]" \
+                           "[preceding-sibling::p" \
+                           "[starts-with(., 'Editor')]]" \
+                           "[following-sibling::p" \
+                           "[contains(@class, 'Title')]]"
+                   page.xpath(xpath).map do |p|
+                     create_contribution_info(p, "editor")
+                   end.compact
+                 else
+                   parse_editors_from_text
+                 end
+        @errors[:editors] &&= result.empty?
+        result
+      end
+
+      def create_contribution_info(person_node, type, description = [])
+        name = person_node.text.match(/^[^(]+/).to_s.strip
+        return nil if name.empty? || !name.match?(/\A\p{L}/) ||
+                      name.match?(%r{\A(?:https?://|urn:)})
+
+        email, org = person_node.xpath ".//a[@href]"
+        entity = create_person name, email, org
+        desc = description.map { |d| Bib::LocalizedMarkedUpString.new(content: d) }
+        role = Bib::Contributor::Role.new(type: type, description: desc)
+        Bib::Contributor.new(role: [role], person: entity)
+      end
+
+      def create_person(name, email = nil, org = nil)
+        forename, surname = name.split
+        fn = Bib::FullNameType::Forename.new(content: forename, language: "en",
+                                             script: "Latn")
+        sn = Bib::LocalizedString.new(content: surname, language: "en",
+                                      script: "Latn")
+        fullname = Bib::FullName.new(surname: sn, forename: [fn])
+        Bib::Person.new(name: fullname, email: person_email(email),
+                        affiliation: person_affiliation(org))
+      end
+
+      def person_email(email)
+        return [] unless email
+
+        href = email[:href]
+        if href.start_with?("mailto:")
+          [href.split(":")[1]]
+        elsif (cf_email = email.at(".//span[@data-cfemail]"))
+          decoded = decode_cf_email(cf_email["data-cfemail"])
+          return [] if decoded.empty?
+
+          # Cloudflare obfuscates ASCII email characters in the data-cfemail
+          # span but leaves non-ASCII characters (e.g. the Latin "fl" ligature
+          # U+FB02) as plain text outside the span. Concatenate any sibling
+          # text and NFKC-normalize so ligatures become their ASCII equivalent.
+          prefix = cf_email.xpath("./preceding-sibling::node()").map(&:text).join
+          suffix = cf_email.xpath("./following-sibling::node()").map(&:text).join
+          [(prefix + decoded + suffix).unicode_normalize(:nfkc)]
+        else
+          []
+        end
+      end
+
+      def decode_cf_email(encoded)
+        bytes = [encoded].pack("H*").bytes
+        key = bytes.first
+        bytes[1..].map { |b| (b ^ key).chr }.join
+      end
+
+      def person_affiliation(org)
+        return [] unless org
+
+        org_name = org.text.gsub(/[\r\n]+/, " ")
+        organization = Bib::Organization.new(
+          name: [Bib::TypedLocalizedString.new(content: org_name)],
+          uri: [Bib::Uri.new(type: "uri", content: org[:href])],
+        )
+        [Bib::Affiliation.new(organization: organization)]
+      end
+
+      #
+      # Parse document identifier specification.
+      #
+      # @param [String] num document number
+      #
+      # @return [String] document identifier with specification if needed
+      #
+      def parse_spec(num)
+        case text
+        when /OASIS Project Specification (\d+)/ then "#{num}-PS#{$1}"
+        when /Committee Specification (\d+)/ then "#{num}-CS#{$1}"
+        else num
+        end
+      end
+
+      #
+      # Parse document identifier part.
+      #
+      # @param [String] docid document identifier
+      #
+      # @return [String] document identifier with part if needed
+      #
+      def parse_part(docid)
+        return docid if docid.match?(/(?:Part|Pt)\d+/i)
+
+        case title
+        when /Part\s(\d+)/ then "#{docid}-Pt#{$1}"
+        else docid
+        end
+      end
+
+      #
+      # Parse document identifier errata.
+      #
+      # @param [String] id document identifier
+      #
+      # @return [String] document identifier with errata if needed
+      #
+      def parse_errata(id)
+        return id.sub("errata", "Errata") if id.match?(/errata\d+/i)
+
+        case title
+        when /Plus\sErrata\s(\d+)/ then "#{id}-plus-Errata#{$1}"
+        when /Errata\s(\d+)/ then "#{id}-Errata#{$1}"
+        else id
+        end
+      end
+
+      #
+      # Parse document identifier.
+      #
+      # @return [Array<Relaton::Oasis::Docidentifier>] document identifier
+      #
+      def parse_docid
+        id = "OASIS #{parse_docnumber}"
+        docid = Docidentifier.new(type: "OASIS", content: id, primary: true)
+        record_unparseable_id docid
+        result = [docid]
+        @errors[:docid] &&= result.empty?
+        result
+      end
+
+      #
+      # Surface an id pubid could not parse through the shared error-reporting
+      # machinery, so the crawl's "Error fetching documents" GitHub issue names
+      # it instead of dropping it silently in the action log.
+      #
+      # `Core::DataFetcher#report_errors` logs a **String** value verbatim as
+      # the message (a boolean means "this field failed for every record" and
+      # takes its message from the key), and the parsers share the fetcher's own
+      # `@errors` hash, so an entry written here reaches the issue unchanged.
+      #
+      # **Why here and not only in the fetcher.** W3C, 3GPP and ISO record this
+      # at index time, where the output file is known. OASIS records it at build
+      # time as well, because this is the one place EVERY id is built — the part
+      # ids that only ever become a relation's `formattedref`
+      # (`DataParser#parse_relation`, `DataPartParser#parse_relation`) never
+      # reach `DataFetcher#save_doc`, so an index-time hook alone cannot see
+      # them. `DataFetcher#add_to_index` keys its own entry on the same id
+      # string, so a document that IS saved reports once, through the fetcher's
+      # message, which additionally names the file.
+      #
+      # The docidentifier is returned either way: the document is still written,
+      # it is only left out of the index — `Relaton::Index` rejects the WHOLE
+      # index if a single row fails to deserialize.
+      #
+      # @param docid [Relaton::Oasis::Docidentifier] the id just built
+      #
+      def record_unparseable_id(docid)
+        return if docid.pubid
+
+        content = docid.content.to_s
+        @errors[content] = "Unparseable primary id `#{content}` was not indexed"
+      end
+
+      #
+      # Parse document type.
+      #
+      # @return [Doctype] document type
+      #
+      def parse_doctype
+        type = case text
+               when /OASIS Project Specification/, /Committee Specification/
+                 "specification"
+               when /Technical Memorandum/ then "memorandum"
+               when /Technical Resolution/ then "resolution"
+               else "standard"
+               end
+        result = Doctype.new(content: type)
+        @errors[:doctype] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse technology area.
+      #
+      # @return [Array<String>] technology areas
+      #
+      def parse_technology_area(node)
+        xpath = "./summary/div/div" \
+                "/ul[@class='technology-areas__list']/li/a"
+        result = node.xpath(xpath).map do |ta|
+          ta.text.strip.gsub(/\s/, "-")
+            .sub("development", "Development")
+        end
+        @errors[:technology_area] &&= result.empty?
+        result
+      end
+
+      def create_ext
+        Ext.new(
+          doctype: parse_doctype,
+          flavor: "oasis",
+          technology_area: parse_technology_area,
+        )
+      end
+    end
+  end
+end

data/lib/relaton/oasis/data_part_parser.rb

@@ -0,0 +1,239 @@
+module Relaton
+  module Oasis
+    # Parser for OASIS part document.
+    class DataPartParser
+      include DataParserUtils
+
+      #
+      # Initialize parser.
+      #
+      # @param [Nokogiri::HTML::Element] node document node
+      #
+      def initialize(node, errors = {}, agent: nil)
+        @node = node
+        @errors = errors
+        @agent = agent
+      end
+
+      def text
+        return @text if @text
+
+        sibling_xpath = "./strong/following-sibling::text()" \
+                        "|./span[strong]/following-sibling::text()"
+        if @node.at(sibling_xpath)
+          nodes_xpath = "./strong/following-sibling::node()" \
+                        "|./span[strong]/following-sibling::node()"
+          @text = @node.xpath(nodes_xpath).text.strip
+        else
+          @text = @node.xpath("./following-sibling::p[1][em]").text.strip
+        end
+      end
+
+      def title
+        return @title if @title
+
+        xpath = "./span[@class='citationTitle' " \
+                "or @class='citeTitle']|./em|./i"
+        t = @node.at(xpath)
+        @title = if t
+                   t.text
+                 else
+                   m = text.match(/(?<content>.+)\s(?:Edited|\d{2}\s\w+\d{4})/)
+                   m ? m[:content] : text
+                 end.strip
+      end
+
+      #
+      # Parse document.
+      #
+      # @return [ItemData] bibliographic item data
+      #
+      def parse # rubocop:disable Metrics/MethodLength
+        ItemData.new(
+          type: "standard",
+          title: parse_title,
+          docidentifier: parse_docid,
+          source: parse_link,
+          docnumber: parse_docnumber,
+          date: parse_date,
+          abstract: parse_abstract,
+          language: ["en"],
+          script: ["Latn"],
+          relation: parse_relation,
+          contributor: parse_contributor,
+          ext: create_ext,
+        )
+      end
+
+      #
+      # Parse title.
+      #
+      # @return [Array<Bib::Title>] title
+      #
+      def parse_title
+        result = [Bib::Title.new(type: "main", content: title,
+                                 language: "en", script: "Latn")]
+        @errors[:part_title] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse document number.
+      #
+      # @return [String] document number
+      #
+      def parse_docnumber
+        ref = @node.at("./span/strong|./strong|./b/span")
+        num = ref.text.match(/[^\[\]]+/).to_s
+        id = parse_errata(num)
+        # some part refs need "Pt" to distinguish from root doc
+        id += "-Pt" if %w[CMIS-v1.1 DocBook-5.0 XACML-V3.0 mqtt-v3.1.1
+                          OData-JSON-Format-v4.0].include?(id)
+        result = parse_part parse_spec id
+        @errors[:part_docnumber] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse link.
+      #
+      # @return [Array<Bib::Uri>] link
+      #
+      def parse_link
+        result = [Bib::Uri.new(type: "src", content: link_node[:href])]
+        @errors[:part_link] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse date.
+      #
+      # @return [Array<Bib::Date>] bibliographic dates
+      #
+      def parse_date
+        match = text.match(/(?<on>\d{1,2}\s\w+\s\d{4})/)
+        result = match ? [Bib::Date.new(at: Date.parse(match[:on]).to_s, type: "issued")] : []
+        @errors[:part_date] &&= result.empty?
+        result
+      end
+
+      def parse_abstract # rubocop:disable Metrics/MethodLength
+        result = if page
+                   xpath = "//p[preceding-sibling::p" \
+                           "[starts-with(., 'Abstract')]][1]"
+                   page.xpath(xpath).map do |p|
+                     cnt = p.text.gsub(/[\r\n]+/, " ").strip
+                     Bib::Abstract.new(
+                       content: cnt, language: "en", script: "Latn",
+                     )
+                   end
+                 else
+                   []
+                 end
+        @errors[:part_abstract] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse editorial group as contributors.
+      #
+      # @return [Array<Bib::Contributor>] editorial group contributors
+      #
+      def parse_editorialgroup_contributor # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        result = if page
+                   xpath = "//p[preceding-sibling::p" \
+                           "[starts-with(., 'Technical')]][1]//a"
+                   tcs = page.xpath(xpath)
+                   if tcs.empty?
+                     []
+                   else
+                     subdivisions = tcs.map do |a|
+                       name = [Bib::TypedLocalizedString.new(
+                         content: a.text.strip,
+                       )]
+                       Bib::Subdivision.new(
+                         type: "technical-committee", name: name,
+                       )
+                     end
+                     org = Bib::Organization.new(
+                       name: [Bib::TypedLocalizedString.new(
+                         content: "OASIS",
+                       )],
+                       subdivision: subdivisions,
+                     )
+                     desc = [Bib::LocalizedMarkedUpString.new(
+                       content: "committee",
+                     )]
+                     role = Bib::Contributor::Role.new(
+                       type: "author", description: desc,
+                     )
+                     [Bib::Contributor.new(organization: org,
+                                           role: [role])]
+                   end
+                 else
+                   []
+                 end
+        @errors[:part_editorialgroup_contributor] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse relation.
+      #
+      # @return [Array<Bib::Relation>] document relations
+      #
+      def parse_relation
+        parser = DataParser.new(@node.at("./ancestor::details"), @errors, agent: @agent)
+        fref = parser.parse_docid[0].content
+        bib = ItemData.new(formattedref: Bib::Formattedref.new(content: fref))
+        result = [Bib::Relation.new(type: "partOf", bibitem: bib)]
+        @errors[:part_relation] &&= result.empty?
+        result
+      end
+
+      def parse_authorizer # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        result = if page
+                   xpath = "//p[preceding-sibling::p" \
+                           "[starts-with(., 'Technical')]][1]//a"
+                   page.xpath(xpath).map do |a|
+                     org_name = a.text.gsub(/[\r\n]+/, " ").strip
+                     org = Bib::Organization.new(
+                       name: [Bib::TypedLocalizedString.new(
+                         content: org_name,
+                       )],
+                       uri: [Bib::Uri.new(type: "uri",
+                                          content: a[:href])],
+                     )
+                     desc = [Bib::LocalizedMarkedUpString.new(
+                       content: "Committee",
+                     )]
+                     role = Bib::Contributor::Role.new(
+                       type: "authorizer", description: desc,
+                     )
+                     Bib::Contributor.new(organization: org,
+                                          role: [role])
+                   end
+                 else
+                   []
+                 end
+        @errors[:part_authorizer] &&= result.empty?
+        result
+      end
+
+      def link_node
+        @link_node = @node.at("./a|./following-sibling::p[1]/a")
+      end
+
+      #
+      # Parse technology area.
+      #
+      # @return [Array<String>] technology areas
+      #
+      def parse_technology_area
+        result = super(@node.at("./ancestor::details"))
+        @errors[:part_technology_area] &&= result.empty?
+        result
+      end
+    end
+  end
+end

data/lib/relaton/oasis/docidentifier.rb

@@ -0,0 +1,54 @@
+module Relaton
+  module Oasis
+    # Structured OASIS document identifier. The docid string is the printed
+    # form, which carries the "OASIS " publisher token
+    # (e.g. "OASIS OSLC-CoreShapes-3.0-PS01-Pt8"), and it is parsed into a
+    # `Pubid::Oasis::Identifier` kept in `@pubid` while the lutaml `content`
+    # attribute stays a plain string for serialization.
+    #
+    # Parsing is **soft**: `content=` lazily requires pubid and rescues
+    # `LoadError`/`StandardError`, so a missing pubid gem or non-OASIS content
+    # leaves `@pubid` nil rather than raising. A reference without the publisher
+    # token ("amqp-core") is not an OASIS printed id, so it leaves `@pubid` nil
+    # too; `DataFetcher` always builds the token-carrying form.
+    #
+    # ## Why the three mutators stay no-ops
+    #
+    # `Pubid::Oasis::Renderer` echoes the verbatim slug held in `original`, so
+    # clearing `part`, `version` or `stage` cannot change the printed id — the
+    # mutation would be invisible, and rewriting `original` would invent a
+    # reference form OASIS does not publish. OASIS slugs are free-form with an
+    # inconsistent internal structure, and no OASIS citation drops a part or a
+    # version; there is no "all parts" or "most recent" spelling to render.
+    #
+    # So this class deliberately does NOT override `remove_part!`,
+    # `remove_date!` or `to_all_parts!`. Since `dfbd26c72` the `Bib::Docidentifier`
+    # defaults are no-ops rather than `NotImplementedError` raises, so
+    # `Bib::ItemData#to_all_parts` and `#to_most_recent_reference` return the
+    # item unchanged instead of blowing up. That is the honest OASIS answer.
+    # (Contrast `Relaton::Ogc::Docidentifier`, whose components really do
+    # render, so its mutators re-render the content.)
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      def initialize(attrs = {}, options = {})
+        pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
+        attrs[:content] ||= pubid.to_s if pubid
+        super
+        @pubid = pubid if pubid
+      end
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          require "pubid"
+          ::Pubid::Oasis::Identifier.parse(value.to_s)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/relaton/oasis/doctype.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Oasis
+    class Doctype <  Bib::Doctype
+      attribute :content, :string, values: %w[specification memorandum resolution standard]
+    end
+  end
+end

data/lib/relaton/oasis/ext.rb

@@ -0,0 +1,19 @@
+require_relative "doctype"
+
+module Relaton
+  module Oasis
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :technology_area, :string, collection: true, values: %w[
+        Cloud Content-Technologies Cybersecurity e-Business eGov/Legal Emergency-Management
+        Energy Information-Modeling IoT Lifecycle-Integration Localization Messaging
+        Privacy/Identity Security SOA Web-Services Software-Development Virtualization
+      ]
+
+      xml { map_element "technology-area", to: :technology_area }
+      key_value { map "technology_area", to: :technology_area }
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-oasis"]
+    end
+  end
+end

data/lib/relaton/oasis/item.rb

@@ -0,0 +1,14 @@
+require_relative "docidentifier"
+require_relative "ext"
+
+module Relaton
+  module Oasis
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier, collection: true,
+                                               initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/oasis/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Oasis
+    class ItemData < Bib::ItemData
+    end
+  end
+end