relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/oasis/data_fetcher.rb

@@ -0,0 +1,130 @@
+require_relative "../oasis"
+require_relative "browser_agent"
+require_relative "data_parser_utils"
+require_relative "data_parser"
+require_relative "data_part_parser"
+
+module Relaton
+  module Oasis
+    class DataFetcher < Core::DataFetcher
+      STANDARDS_URL = "https://www.oasis-open.org/standards/".freeze
+      RETRIABLE_ERRORS = [
+        SocketError,
+        Ferrum::TimeoutError,
+        Ferrum::PendingConnectionsError,
+        Ferrum::StatusError,
+      ].freeze
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def fetch(_source = nil)
+        doc = with_retry { agent.get(STANDARDS_URL) }
+        doc.xpath("//details").map do |item|
+          save_doc DataParser.new(item, @errors, agent: agent).parse
+          fetch_parts item
+        end
+        index.save
+        report_errors
+      ensure
+        @agent&.quit
+      end
+
+      private
+
+      def agent
+        @agent ||= BrowserAgent.new
+      end
+
+      def with_retry
+        tries = 0
+        begin
+          tries += 1
+          yield
+        rescue *RETRIABLE_ERRORS => e
+          retry if tries < 4
+          raise e
+        end
+      end
+
+      # `pubid_class:` on the producer too: FileIO#save only calls `to_hash`
+      # for instances of it, so without it the crawl writes v1-shaped rows
+      # under a v2 name, silently.
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :oasis, file: "#{INDEXFILE}.yaml",
+          pubid_class: ::Pubid::Oasis::Identifier
+        )
+      end
+
+      def fetch_parts(item)
+        xpath = "./div/div/div[contains(@class, " \
+                "'standard__grid--cite-as')]" \
+                "/p[strong or span/strong]"
+        parts = item.xpath(xpath)
+        return unless parts.size > 1
+
+        parts.each do |part|
+          save_doc DataPartParser.new(part, @errors, agent: agent).parse
+        end
+      end
+
+      def save_doc(doc) # rubocop:disable Metrics/AbcSize
+        id = doc.docidentifier.find(&:primary) || doc.docidentifier.first
+        # Distinct docids can sanitize to one filename; take a path of our own
+        # rather than overwriting the other document (Core#unique_output_file).
+        file = unique_output_file(id.content)
+        if @files.include? file
+          # Same reserved path == same docid: a genuine duplicate. Checked FIRST,
+          # because a disambiguated path stays != output_file forever.
+          Util.warn "File #{file} already exists. Document: #{id.content}"
+        elsif file != output_file(id.content)
+          Util.warn "File #{output_file id.content} already exists. " \
+                    "Document: #{id.content}. Writing #{file} instead."
+        end
+        @files << file
+        add_to_index id, file
+        File.write file, serialize(doc), encoding: "UTF-8"
+      end
+
+      #
+      # Index the document, or record why it could not be indexed.
+      #
+      # An id pubid cannot parse is recorded in `@errors` — the inherited
+      # `report_errors` logs a String value as the message — and the row is
+      # skipped rather than indexed unparsed: `Relaton::Index` rejects the WHOLE
+      # index if a single row fails to deserialize, and its sort calls
+      # `.root.number` on every id. The data file is still written by the
+      # caller, so the document is unindexed, never lost. (The ECMA/W3C/3GPP
+      # shape.)
+      #
+      # `DataParserUtils#record_unparseable_id` already recorded this id when it
+      # was built, under the SAME key — the id string. This entry overwrites it
+      # with the same sentence plus the output file, so a saved document reports
+      # one line, the more informative one. Keep the two keys identical.
+      #
+      # @param id [Relaton::Oasis::Docidentifier] primary document identifier
+      # @param file [String] path the document is written to
+      #
+      def add_to_index(id, file)
+        return index.add_or_update(id.pubid, file) if id.pubid
+
+        @errors[id.content.to_s] =
+          "Unparseable primary id `#{id.content}` was not indexed (#{file})"
+      end
+
+      def to_xml(bib)
+        bib.to_xml(bibdata: true)
+      end
+
+      def to_yaml(bib)
+        bib.to_yaml
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+    end
+  end
+end

data/lib/relaton/oasis/data_parser.rb

@@ -0,0 +1,305 @@
+module Relaton
+  module Oasis
+    # Parser for OASIS document.
+    class DataParser
+      include DataParserUtils
+
+      #
+      # Initialize parser.
+      #
+      # @param [Nokogiri::HTML::Element] node document node
+      #
+      def initialize(node, errors = {}, agent: nil)
+        @node = node
+        @errors = errors
+        @agent = agent
+      end
+
+      def title
+        @title ||= @node.at("./summary/div/h2").text
+      end
+
+      def text
+        xpath = "./div/div/div[contains(@class, " \
+                "'standard__grid--cite-as')]" \
+                "/p[em or i or a or span]"
+        @text ||= @node.at(xpath)&.text&.strip
+      end
+
+      #
+      # Parse document.
+      #
+      # @return [ItemData] bibliographic item data
+      #
+      def parse # rubocop:disable Metrics/MethodLength
+        ItemData.new(
+          type: "standard",
+          title: parse_title,
+          docidentifier: parse_docid,
+          source: parse_link,
+          docnumber: parse_docnumber,
+          date: parse_date,
+          contributor: parse_contributor,
+          abstract: parse_abstract,
+          language: ["en"],
+          script: ["Latn"],
+          relation: parse_relation,
+          ext: create_ext,
+        )
+      end
+
+      #
+      # Parse title.
+      #
+      # @return [Array<Bib::Title>] title
+      #
+      def parse_title
+        result = [Bib::Title.new(type: "main", content: title, language: "en",
+                                 script: "Latn")]
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse date.
+      #
+      # @return [Array<Bib::Date>] date
+      #
+      def parse_date
+        xpath = "./summary/div/time[@class='standard__date']"
+        result = @node.xpath(xpath).map do |d|
+          date_str = d.text.match(/\d{2}\s\w+\s\d{4}/).to_s
+          date = Date.parse(date_str).to_s
+          Bib::Date.new(at: date, type: "issued")
+        end
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse abstract.
+      #
+      # @return [Array<Bib::LocalizedMarkedUpString>] abstract
+      #
+      def parse_abstract
+        c = @node.xpath(
+          "./summary/div/div[@class='standard__description']/p",
+        ).map { |a| a.text.gsub(/[\n\t]+/, " ").strip }.join("\n")
+        result = if c.empty?
+                   []
+                 else
+                   [Bib::Abstract.new(
+                     content: c, language: "en", script: "Latn",
+                   )]
+                 end
+        @errors[:abstract] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse editorial group as contributors.
+      #
+      # @return [Array<Bib::Contributor>] editorial group contributors
+      #
+      def parse_editorialgroup_contributor # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        tcs = @node.xpath("./div[@class='standard__details']/a")
+        if tcs.empty?
+          result = []
+        else
+          subdivisions = tcs.map do |a|
+            name = [Bib::TypedLocalizedString.new(content: a.text.strip)]
+            Bib::Subdivision.new(type: "technical-committee",
+                                 name: name)
+          end
+          org = Bib::Organization.new(
+            name: [Bib::TypedLocalizedString.new(content: "OASIS")],
+            subdivision: subdivisions,
+          )
+          desc = [Bib::LocalizedMarkedUpString.new(content: "committee")]
+          role = Bib::Contributor::Role.new(
+            type: "author", description: desc,
+          )
+          result = [Bib::Contributor.new(organization: org, role: [role])]
+        end
+        @errors[:editorialgroup_contributor] &&= result.empty?
+        result
+      end
+
+      def parse_authorizer # rubocop:disable Metrics/MethodLength
+        result = @node.xpath("./div[@class='standard__details']/a").map do |a|
+          org = Bib::Organization.new(
+            name: [Bib::TypedLocalizedString.new(content: a.text.strip)],
+            uri: [Bib::Uri.new(type: "uri", content: a[:href])],
+          )
+          desc = [Bib::LocalizedMarkedUpString.new(content: "Committee")]
+          role = Bib::Contributor::Role.new(type: "authorizer",
+                                            description: desc)
+          Bib::Contributor.new(organization: org, role: [role])
+        end
+        @errors[:authorizer] &&= result.empty?
+        result
+      end
+
+      def link_node
+        xpath = "./div/div/div[contains(@class, " \
+                "'standard__grid--cite-as')]" \
+                "/p[strong or span/strong]/a"
+        @link_node ||= @node.at(xpath)
+      end
+
+      #
+      # Parse relation.
+      #
+      # @return [Array<Bib::Relation>] relation
+      #
+      def parse_relation # rubocop:disable Metrics/MethodLength
+        xpath = "./div/div/div[contains(@class, " \
+                "'standard__grid--cite-as')]" \
+                "/p[strong or span/strong or b/span]"
+        rels = @node.xpath(xpath)
+        result = if rels.size > 1
+                   rels.map do |r|
+                     docid = DataPartParser.new(r, @errors, agent: @agent).parse_docid
+                     bib = ItemData.new(formattedref: Bib::Formattedref.new(content: docid[0].content))
+                     Bib::Relation.new(type: "hasPart", bibitem: bib)
+                   end
+                 else
+                   []
+                 end
+        @errors[:relation] &&= result.empty?
+        result
+      end
+
+      #
+      # Look for "Cite as" references.
+      #
+      # @return [Array<String>] document part references
+      #
+      def document_part_refs
+        @node.css(
+          ".standard__grid--cite-as > p > strong",
+          "span.Refterm", "span.abbrev", "span.citationLabel > strong"
+        ).map { |p| p.text.gsub(/^\[{1,2}|\]$/, "").strip }
+      end
+
+      def parse_link # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        result = if parts.size > 1
+                   []
+                 else
+                   links.map do |l|
+                     type = l[:href].match(/\.(\w+)$/)&.captures&.first
+                     type ||= "src"
+                     type.sub!("docx", "doc")
+                     type.sub!("html", "src")
+                     Bib::Uri.new(type: type, content: l[:href])
+                   end
+                 end
+        @errors[:link] &&= result.empty?
+        result
+      end
+
+      def parts
+        xpath = "./div/div/div[contains(@class, " \
+                "'standard__grid--cite-as')]" \
+                "/p[strong or span/strong]"
+        @parts ||= @node.xpath(xpath)
+      end
+
+      def links
+        l = @node.xpath("./div/div/div[1]/p[1]/a[@href]")
+        l = @node.xpath("./div/div/div[1]/p[2]/a[@href]") if l.empty?
+        l
+      end
+
+      #
+      # Parse document number.
+      #
+      # @return [String] document number
+      #
+      def parse_docnumber
+        parts = document_part_refs
+        result = case parts.size
+                 when 0
+                   txt = @node.at("./summary/div/h2").text
+                   parse_spec title_to_docid(txt)
+                 when 1 then parse_part parse_spec(parts[0])
+                 else parts_to_docid parts
+                 end
+        @errors[:docnumber] &&= result.nil?
+        result
+      end
+
+      #
+      # Create document identifier from parts references.
+      #
+      # @param [Array<String>] parts parts references
+      #
+      # @return [String] document identifier
+      #
+      def parts_to_docid(parts) # rubocop:disable Metrics/AbcSize
+        id = parts[1..].each_with_object(parts[0].split("-")) do |part, acc|
+          chunks = part.split "-"
+          chunks.each.with_index do |chunk, idx|
+            unless chunk.casecmp(acc[idx])&.zero?
+              acc.slice!(idx..-1)
+              break
+            end
+          end
+        end.join("-")
+        parse_part parse_spec(id)
+      end
+
+      #
+      # Create document identifier from title.
+      #
+      # @param [String] title title
+      #
+      # @return [String] document identifier
+      #
+      def title_to_docid(title) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        abbrs = title.scan(/(?<=\()[^)]+(?=\))/)
+        if abbrs.any?
+          id = abbrs.map { |abbr| abbr.split.join("-") }.join "-"
+          /(?:Version\s|v)(?<ver>[\d.]+)/ =~ title
+          id += "-v#{ver}" if ver
+          /(?<eb>ebXML|ebMS)/ =~ title
+          id = "#{eb}-#{id}" if eb
+          id
+        else
+          series_end = false
+          title.sub(/\s\[OASIS\s\d+\]$/,
+                    "").split(/[,:]?\s|-|(?<=[a-z])(?=[A-Z][a-z])/)
+            .each_with_object([""]) do |word, acc|
+              if word =~ /^v[\d.]+/
+                acc << $MATCH.to_s
+                series_end = true
+              elsif word.match?(/^Version/)
+                acc << "v"
+                series_end = false
+              elsif word.match?(/^\d|ebXML|ebMS/)
+                series_end ? acc << word : acc[-1] += word
+                series_end = true
+              elsif word.match?(/^\w+$/) && word == word.upcase
+                series_end ? acc << word : acc[-1] = word
+                series_end = true
+              elsif word.match?(/[A-Z]+[a-z]+/)
+                series_end ? acc << word[0] : acc[-1] += word[0]
+                series_end = false
+              end
+          end.join "-"
+        end
+      end
+
+      #
+      # Parse technology areas.
+      #
+      # @return [Array<String>] technology areas
+      #
+      def parse_technology_area
+        result = super(@node)
+        @errors[:technology_area] &&= result.empty?
+        result
+      end
+    end
+  end
+end