relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/oasis/data_fetcher.rb

@@ -0,0 +1,94 @@
+require_relative "../oasis"
+require_relative "browser_agent"
+require_relative "data_parser_utils"
+require_relative "data_parser"
+require_relative "data_part_parser"
+
+module Relaton
+  module Oasis
+    class DataFetcher < Core::DataFetcher
+      STANDARDS_URL = "https://www.oasis-open.org/standards/".freeze
+      RETRIABLE_ERRORS = [
+        SocketError,
+        Ferrum::TimeoutError,
+        Ferrum::PendingConnectionsError,
+        Ferrum::StatusError,
+      ].freeze
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def fetch(_source = nil)
+        doc = with_retry { agent.get(STANDARDS_URL) }
+        doc.xpath("//details").map do |item|
+          save_doc DataParser.new(item, @errors, agent: agent).parse
+          fetch_parts item
+        end
+        index.save
+        report_errors
+      ensure
+        @agent&.quit
+      end
+
+      private
+
+      def agent
+        @agent ||= BrowserAgent.new
+      end
+
+      def with_retry
+        tries = 0
+        begin
+          tries += 1
+          yield
+        rescue *RETRIABLE_ERRORS => e
+          retry if tries < 4
+          raise e
+        end
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :oasis, file: "#{INDEXFILE}.yaml"
+        )
+      end
+
+      def fetch_parts(item)
+        xpath = "./div/div/div[contains(@class, " \
+                "'standard__grid--cite-as')]" \
+                "/p[strong or span/strong]"
+        parts = item.xpath(xpath)
+        return unless parts.size > 1
+
+        parts.each do |part|
+          save_doc DataPartParser.new(part, @errors, agent: agent).parse
+        end
+      end
+
+      def save_doc(doc) # rubocop:disable Metrics/AbcSize
+        id = doc.docidentifier.find(&:primary) || doc.docidentifier.first
+        file = output_file(id.content)
+        if @files.include? file
+          Util.warn "File #{file} already exists. Document: #{id.content}"
+        else
+          @files << file
+        end
+        index.add_or_update id.content, file
+        File.write file, serialize(doc), encoding: "UTF-8"
+      end
+
+      def to_xml(bib)
+        bib.to_xml(bibdata: true)
+      end
+
+      def to_yaml(bib)
+        bib.to_yaml
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+    end
+  end
+end

data/lib/relaton/oasis/data_parser.rb

@@ -0,0 +1,305 @@
+module Relaton
+  module Oasis
+    # Parser for OASIS document.
+    class DataParser
+      include DataParserUtils
+
+      #
+      # Initialize parser.
+      #
+      # @param [Nokogiri::HTML::Element] node document node
+      #
+      def initialize(node, errors = {}, agent: nil)
+        @node = node
+        @errors = errors
+        @agent = agent
+      end
+
+      def title
+        @title ||= @node.at("./summary/div/h2").text
+      end
+
+      def text
+        xpath = "./div/div/div[contains(@class, " \
+                "'standard__grid--cite-as')]" \
+                "/p[em or i or a or span]"
+        @text ||= @node.at(xpath)&.text&.strip
+      end
+
+      #
+      # Parse document.
+      #
+      # @return [ItemData] bibliographic item data
+      #
+      def parse # rubocop:disable Metrics/MethodLength
+        ItemData.new(
+          type: "standard",
+          title: parse_title,
+          docidentifier: parse_docid,
+          source: parse_link,
+          docnumber: parse_docnumber,
+          date: parse_date,
+          contributor: parse_contributor,
+          abstract: parse_abstract,
+          language: ["en"],
+          script: ["Latn"],
+          relation: parse_relation,
+          ext: create_ext,
+        )
+      end
+
+      #
+      # Parse title.
+      #
+      # @return [Array<Bib::Title>] title
+      #
+      def parse_title
+        result = [Bib::Title.new(type: "main", content: title, language: "en",
+                                 script: "Latn")]
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse date.
+      #
+      # @return [Array<Bib::Date>] date
+      #
+      def parse_date
+        xpath = "./summary/div/time[@class='standard__date']"
+        result = @node.xpath(xpath).map do |d|
+          date_str = d.text.match(/\d{2}\s\w+\s\d{4}/).to_s
+          date = Date.parse(date_str).to_s
+          Bib::Date.new(at: date, type: "issued")
+        end
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse abstract.
+      #
+      # @return [Array<Bib::LocalizedMarkedUpString>] abstract
+      #
+      def parse_abstract
+        c = @node.xpath(
+          "./summary/div/div[@class='standard__description']/p",
+        ).map { |a| a.text.gsub(/[\n\t]+/, " ").strip }.join("\n")
+        result = if c.empty?
+                   []
+                 else
+                   [Bib::Abstract.new(
+                     content: c, language: "en", script: "Latn",
+                   )]
+                 end
+        @errors[:abstract] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse editorial group as contributors.
+      #
+      # @return [Array<Bib::Contributor>] editorial group contributors
+      #
+      def parse_editorialgroup_contributor # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        tcs = @node.xpath("./div[@class='standard__details']/a")
+        if tcs.empty?
+          result = []
+        else
+          subdivisions = tcs.map do |a|
+            name = [Bib::TypedLocalizedString.new(content: a.text.strip)]
+            Bib::Subdivision.new(type: "technical-committee",
+                                 name: name)
+          end
+          org = Bib::Organization.new(
+            name: [Bib::TypedLocalizedString.new(content: "OASIS")],
+            subdivision: subdivisions,
+          )
+          desc = [Bib::LocalizedMarkedUpString.new(content: "committee")]
+          role = Bib::Contributor::Role.new(
+            type: "author", description: desc,
+          )
+          result = [Bib::Contributor.new(organization: org, role: [role])]
+        end
+        @errors[:editorialgroup_contributor] &&= result.empty?
+        result
+      end
+
+      def parse_authorizer # rubocop:disable Metrics/MethodLength
+        result = @node.xpath("./div[@class='standard__details']/a").map do |a|
+          org = Bib::Organization.new(
+            name: [Bib::TypedLocalizedString.new(content: a.text.strip)],
+            uri: [Bib::Uri.new(type: "uri", content: a[:href])],
+          )
+          desc = [Bib::LocalizedMarkedUpString.new(content: "Committee")]
+          role = Bib::Contributor::Role.new(type: "authorizer",
+                                            description: desc)
+          Bib::Contributor.new(organization: org, role: [role])
+        end
+        @errors[:authorizer] &&= result.empty?
+        result
+      end
+
+      def link_node
+        xpath = "./div/div/div[contains(@class, " \
+                "'standard__grid--cite-as')]" \
+                "/p[strong or span/strong]/a"
+        @link_node ||= @node.at(xpath)
+      end
+
+      #
+      # Parse relation.
+      #
+      # @return [Array<Bib::Relation>] relation
+      #
+      def parse_relation # rubocop:disable Metrics/MethodLength
+        xpath = "./div/div/div[contains(@class, " \
+                "'standard__grid--cite-as')]" \
+                "/p[strong or span/strong or b/span]"
+        rels = @node.xpath(xpath)
+        result = if rels.size > 1
+                   rels.map do |r|
+                     docid = DataPartParser.new(r, @errors, agent: @agent).parse_docid
+                     bib = ItemData.new(formattedref: Bib::Formattedref.new(content: docid[0].content))
+                     Bib::Relation.new(type: "hasPart", bibitem: bib)
+                   end
+                 else
+                   []
+                 end
+        @errors[:relation] &&= result.empty?
+        result
+      end
+
+      #
+      # Look for "Cite as" references.
+      #
+      # @return [Array<String>] document part references
+      #
+      def document_part_refs
+        @node.css(
+          ".standard__grid--cite-as > p > strong",
+          "span.Refterm", "span.abbrev", "span.citationLabel > strong"
+        ).map { |p| p.text.gsub(/^\[{1,2}|\]$/, "").strip }
+      end
+
+      def parse_link # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        result = if parts.size > 1
+                   []
+                 else
+                   links.map do |l|
+                     type = l[:href].match(/\.(\w+)$/)&.captures&.first
+                     type ||= "src"
+                     type.sub!("docx", "doc")
+                     type.sub!("html", "src")
+                     Bib::Uri.new(type: type, content: l[:href])
+                   end
+                 end
+        @errors[:link] &&= result.empty?
+        result
+      end
+
+      def parts
+        xpath = "./div/div/div[contains(@class, " \
+                "'standard__grid--cite-as')]" \
+                "/p[strong or span/strong]"
+        @parts ||= @node.xpath(xpath)
+      end
+
+      def links
+        l = @node.xpath("./div/div/div[1]/p[1]/a[@href]")
+        l = @node.xpath("./div/div/div[1]/p[2]/a[@href]") if l.empty?
+        l
+      end
+
+      #
+      # Parse document number.
+      #
+      # @return [String] document number
+      #
+      def parse_docnumber
+        parts = document_part_refs
+        result = case parts.size
+                 when 0
+                   txt = @node.at("./summary/div/h2").text
+                   parse_spec title_to_docid(txt)
+                 when 1 then parse_part parse_spec(parts[0])
+                 else parts_to_docid parts
+                 end
+        @errors[:docnumber] &&= result.nil?
+        result
+      end
+
+      #
+      # Create document identifier from parts references.
+      #
+      # @param [Array<String>] parts parts references
+      #
+      # @return [String] document identifier
+      #
+      def parts_to_docid(parts) # rubocop:disable Metrics/AbcSize
+        id = parts[1..].each_with_object(parts[0].split("-")) do |part, acc|
+          chunks = part.split "-"
+          chunks.each.with_index do |chunk, idx|
+            unless chunk.casecmp(acc[idx])&.zero?
+              acc.slice!(idx..-1)
+              break
+            end
+          end
+        end.join("-")
+        parse_part parse_spec(id)
+      end
+
+      #
+      # Create document identifier from title.
+      #
+      # @param [String] title title
+      #
+      # @return [String] document identifier
+      #
+      def title_to_docid(title) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        abbrs = title.scan(/(?<=\()[^)]+(?=\))/)
+        if abbrs.any?
+          id = abbrs.map { |abbr| abbr.split.join("-") }.join "-"
+          /(?:Version\s|v)(?<ver>[\d.]+)/ =~ title
+          id += "-v#{ver}" if ver
+          /(?<eb>ebXML|ebMS)/ =~ title
+          id = "#{eb}-#{id}" if eb
+          id
+        else
+          series_end = false
+          title.sub(/\s\[OASIS\s\d+\]$/,
+                    "").split(/[,:]?\s|-|(?<=[a-z])(?=[A-Z][a-z])/)
+            .each_with_object([""]) do |word, acc|
+              if word =~ /^v[\d.]+/
+                acc << $MATCH.to_s
+                series_end = true
+              elsif word.match?(/^Version/)
+                acc << "v"
+                series_end = false
+              elsif word.match?(/^\d|ebXML|ebMS/)
+                series_end ? acc << word : acc[-1] += word
+                series_end = true
+              elsif word.match?(/^\w+$/) && word == word.upcase
+                series_end ? acc << word : acc[-1] = word
+                series_end = true
+              elsif word.match?(/[A-Z]+[a-z]+/)
+                series_end ? acc << word[0] : acc[-1] += word[0]
+                series_end = false
+              end
+          end.join "-"
+        end
+      end
+
+      #
+      # Parse technology areas.
+      #
+      # @return [Array<String>] technology areas
+      #
+      def parse_technology_area
+        result = super(@node)
+        @errors[:technology_area] &&= result.empty?
+        result
+      end
+    end
+  end
+end

data/lib/relaton/oasis/data_parser_utils.rb

@@ -0,0 +1,296 @@
+require "mechanize"
+require_relative "browser_agent"
+
+module Relaton
+  module Oasis
+    # Common methods for document and part parsers.
+    module DataParserUtils
+      RETRIABLE_PAGE_ERRORS = [
+        Errno::ETIMEDOUT,
+        Net::OpenTimeout,
+        Ferrum::TimeoutError,
+        Ferrum::PendingConnectionsError,
+        Ferrum::StatusError,
+      ].freeze
+      #
+      # Parse contributor.
+      #
+      # @return [Array<Bib::Contributor>] contributors
+      #
+      def parse_contributor
+        result = publisher_oasis + parse_authorizer +
+          parse_editorialgroup_contributor +
+          parse_chairs + parse_editors
+        @errors[:contributor] &&= result.empty?
+        result
+      end
+
+      def publisher_oasis
+        org = Bib::Organization.new(
+          name: [Bib::TypedLocalizedString.new(content: "OASIS")],
+          uri: [Bib::Uri.new(type: "uri", content: "https://www.oasis-open.org/")],
+        )
+        role = [
+          Bib::Contributor::Role.new(
+            type: "authorizer",
+            description: [Bib::LocalizedMarkedUpString.new(
+              content: "Standards Development Organization",
+            )],
+          ),
+          Bib::Contributor::Role.new(type: "publisher"),
+        ]
+        [Bib::Contributor.new(organization: org, role: role)]
+      end
+
+      def parse_editors_from_text # rubocop:disable Metrics/MethodLength
+        result = if text
+                   text.match(/(?<=Edited\sby\s)[^.]+/).to_s
+                     .split(/,?\sand\s|,\s/).map do |c|
+                     role = [Bib::Contributor::Role.new(type: "editor")]
+                     Bib::Contributor.new(role: role,
+                                          person: create_person(c))
+                   end
+                 else
+                   []
+                 end
+        @errors[:editors] &&= result.empty?
+        result
+      end
+
+      def page
+        return @page if defined? @page
+
+        @page = nil
+        return @page unless link_node && link_node[:href].match?(/\.html$/)
+
+        if @agent
+          doc = retry_page(link_node[:href], @agent)
+          @page = doc if doc && @agent.last_status == 200
+        else
+          # No injected agent (e.g. unit tests with VCR cassettes): fall back
+          # to a Mechanize request — VCR can intercept it.
+          agent = Mechanize.new
+          agent.agent.allowed_error_codes = [403, 404, 503]
+          resp = retry_page(link_node[:href], agent)
+          @page = resp if resp && resp.code == "200"
+        end
+      end
+
+      #
+      # Retry to get page.
+      #
+      # @param [String] url page URL
+      # @param [#get] agent HTTP client responding to #get(url)
+      # @param [Integer] retries number of retries
+      #
+      # @return [Nokogiri::HTML::Document, Mechanize::Page, nil] page or nil
+      #
+      def retry_page(url, agent, retries = 3)
+        sleep 1 # to avoid 429 error
+        agent.get url
+      rescue *RETRIABLE_PAGE_ERRORS => e
+        retry if (retries -= 1).positive?
+        Util.error "Failed to get page `#{url}`\n#{e.message}"
+        nil
+      end
+
+      def parse_chairs # rubocop:disable Metrics/MethodLength
+        result = if page
+                   xpath = "//p[preceding-sibling::p" \
+                           "[starts-with(., 'Chair')]]" \
+                           "[following-sibling::p" \
+                           "[starts-with(., 'Editor')]]"
+                   page.xpath(xpath).map do |p|
+                     create_contribution_info(p, "editor", ["Chair"])
+                   end.compact
+                 else
+                   []
+                 end
+        @errors[:chairs] &&= result.empty?
+        result
+      end
+
+      def parse_editors # rubocop:disable Metrics/MethodLength
+        result = if page
+                   xpath = "//p[contains(@class, 'Contributor')]" \
+                           "[preceding-sibling::p" \
+                           "[starts-with(., 'Editor')]]" \
+                           "[following-sibling::p" \
+                           "[contains(@class, 'Title')]]"
+                   page.xpath(xpath).map do |p|
+                     create_contribution_info(p, "editor")
+                   end.compact
+                 else
+                   parse_editors_from_text
+                 end
+        @errors[:editors] &&= result.empty?
+        result
+      end
+
+      def create_contribution_info(person_node, type, description = [])
+        name = person_node.text.match(/^[^(]+/).to_s.strip
+        return nil if name.empty? || !name.match?(/\A\p{L}/) ||
+                      name.match?(%r{\A(?:https?://|urn:)})
+
+        email, org = person_node.xpath ".//a[@href]"
+        entity = create_person name, email, org
+        desc = description.map { |d| Bib::LocalizedMarkedUpString.new(content: d) }
+        role = Bib::Contributor::Role.new(type: type, description: desc)
+        Bib::Contributor.new(role: [role], person: entity)
+      end
+
+      def create_person(name, email = nil, org = nil)
+        forename, surname = name.split
+        fn = Bib::FullNameType::Forename.new(content: forename, language: "en",
+                                             script: "Latn")
+        sn = Bib::LocalizedString.new(content: surname, language: "en",
+                                      script: "Latn")
+        fullname = Bib::FullName.new(surname: sn, forename: [fn])
+        Bib::Person.new(name: fullname, email: person_email(email),
+                        affiliation: person_affiliation(org))
+      end
+
+      def person_email(email)
+        return [] unless email
+
+        href = email[:href]
+        if href.start_with?("mailto:")
+          [href.split(":")[1]]
+        elsif (cf_email = email.at(".//span[@data-cfemail]"))
+          decoded = decode_cf_email(cf_email["data-cfemail"])
+          return [] if decoded.empty?
+
+          # Cloudflare obfuscates ASCII email characters in the data-cfemail
+          # span but leaves non-ASCII characters (e.g. the Latin "fl" ligature
+          # U+FB02) as plain text outside the span. Concatenate any sibling
+          # text and NFKC-normalize so ligatures become their ASCII equivalent.
+          prefix = cf_email.xpath("./preceding-sibling::node()").map(&:text).join
+          suffix = cf_email.xpath("./following-sibling::node()").map(&:text).join
+          [(prefix + decoded + suffix).unicode_normalize(:nfkc)]
+        else
+          []
+        end
+      end
+
+      def decode_cf_email(encoded)
+        bytes = [encoded].pack("H*").bytes
+        key = bytes.first
+        bytes[1..].map { |b| (b ^ key).chr }.join
+      end
+
+      def person_affiliation(org)
+        return [] unless org
+
+        org_name = org.text.gsub(/[\r\n]+/, " ")
+        organization = Bib::Organization.new(
+          name: [Bib::TypedLocalizedString.new(content: org_name)],
+          uri: [Bib::Uri.new(type: "uri", content: org[:href])],
+        )
+        [Bib::Affiliation.new(organization: organization)]
+      end
+
+      #
+      # Parse document identifier specification.
+      #
+      # @param [String] num document number
+      #
+      # @return [String] document identifier with specification if needed
+      #
+      def parse_spec(num)
+        case text
+        when /OASIS Project Specification (\d+)/ then "#{num}-PS#{$1}"
+        when /Committee Specification (\d+)/ then "#{num}-CS#{$1}"
+        else num
+        end
+      end
+
+      #
+      # Parse document identifier part.
+      #
+      # @param [String] docid document identifier
+      #
+      # @return [String] document identifier with part if needed
+      #
+      def parse_part(docid)
+        return docid if docid.match?(/(?:Part|Pt)\d+/i)
+
+        case title
+        when /Part\s(\d+)/ then "#{docid}-Pt#{$1}"
+        else docid
+        end
+      end
+
+      #
+      # Parse document identifier errata.
+      #
+      # @param [String] id document identifier
+      #
+      # @return [String] document identifier with errata if needed
+      #
+      def parse_errata(id)
+        return id.sub("errata", "Errata") if id.match?(/errata\d+/i)
+
+        case title
+        when /Plus\sErrata\s(\d+)/ then "#{id}-plus-Errata#{$1}"
+        when /Errata\s(\d+)/ then "#{id}-Errata#{$1}"
+        else id
+        end
+      end
+
+      #
+      # Parse document identifier.
+      #
+      # @return [Array<Bib::Docidentifier>] document identifier
+      #
+      def parse_docid
+        id = "OASIS #{parse_docnumber}"
+        result = [Bib::Docidentifier.new(type: "OASIS", content: id,
+                                         primary: true)]
+        @errors[:docid] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse document type.
+      #
+      # @return [Doctype] document type
+      #
+      def parse_doctype
+        type = case text
+               when /OASIS Project Specification/, /Committee Specification/
+                 "specification"
+               when /Technical Memorandum/ then "memorandum"
+               when /Technical Resolution/ then "resolution"
+               else "standard"
+               end
+        result = Doctype.new(content: type)
+        @errors[:doctype] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse technology area.
+      #
+      # @return [Array<String>] technology areas
+      #
+      def parse_technology_area(node)
+        xpath = "./summary/div/div" \
+                "/ul[@class='technology-areas__list']/li/a"
+        result = node.xpath(xpath).map do |ta|
+          ta.text.strip.gsub(/\s/, "-")
+            .sub("development", "Development")
+        end
+        @errors[:technology_area] &&= result.empty?
+        result
+      end
+
+      def create_ext
+        Ext.new(
+          doctype: parse_doctype,
+          flavor: "oasis",
+          technology_area: parse_technology_area,
+        )
+      end
+    end
+  end
+end