relaton-oasis 1.10.0

data/grammars/reqt.rng

@@ -0,0 +1,223 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <!--
+    Presupposes isodoc.rnc, is included in it
+    include "isodoc.rnc" { }
+  -->
+  <define name="requirement">
+    <element name="requirement">
+      <ref name="RequirementType"/>
+    </element>
+  </define>
+  <define name="recommendation">
+    <element name="recommendation">
+      <ref name="RequirementType"/>
+    </element>
+  </define>
+  <define name="permission">
+    <element name="permission">
+      <ref name="RequirementType"/>
+    </element>
+  </define>
+  <define name="RequirementType">
+    <optional>
+      <attribute name="obligation">
+        <ref name="ObligationType"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="unnumbered">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="number"/>
+    </optional>
+    <optional>
+      <attribute name="subsequence"/>
+    </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <attribute name="id">
+      <data type="ID"/>
+    </attribute>
+    <optional>
+      <attribute name="filename"/>
+    </optional>
+    <optional>
+      <attribute name="model"/>
+    </optional>
+    <optional>
+      <attribute name="type"/>
+    </optional>
+    <optional>
+      <attribute name="tag"/>
+    </optional>
+    <optional>
+      <attribute name="multilingual-rendering">
+        <ref name="MultilingualRenderingType"/>
+      </attribute>
+    </optional>
+    <optional>
+      <ref name="reqtitle"/>
+    </optional>
+    <optional>
+      <ref name="label"/>
+    </optional>
+    <zeroOrMore>
+      <ref name="subject"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="reqinherit"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="classification"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <choice>
+        <ref name="measurementtarget"/>
+        <ref name="specification"/>
+        <ref name="verification"/>
+        <ref name="import"/>
+        <ref name="description"/>
+        <ref name="component"/>
+      </choice>
+    </zeroOrMore>
+    <optional>
+      <ref name="reqt_references"/>
+    </optional>
+    <zeroOrMore>
+      <choice>
+        <ref name="requirement"/>
+        <ref name="recommendation"/>
+        <ref name="permission"/>
+      </choice>
+    </zeroOrMore>
+  </define>
+  <define name="reqtitle">
+    <element name="title">
+      <ref name="FormattedString"/>
+    </element>
+  </define>
+  <define name="label">
+    <element name="label">
+      <oneOrMore>
+        <ref name="TextElement"/>
+      </oneOrMore>
+    </element>
+  </define>
+  <define name="subject">
+    <element name="subject">
+      <oneOrMore>
+        <ref name="TextElement"/>
+      </oneOrMore>
+    </element>
+  </define>
+  <define name="reqinherit">
+    <element name="inherit">
+      <oneOrMore>
+        <ref name="TextElement"/>
+      </oneOrMore>
+    </element>
+  </define>
+  <define name="measurementtarget">
+    <element name="measurement-target">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="specification">
+    <element name="specification">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="verification">
+    <element name="verification">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="import">
+    <element name="import">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="description">
+    <element name="description">
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="component">
+    <element name="component">
+      <attribute name="class"/>
+      <ref name="RequirementSubpart"/>
+    </element>
+  </define>
+  <define name="reqt_references">
+    <element name="references">
+      <oneOrMore>
+        <ref name="bibitem"/>
+      </oneOrMore>
+    </element>
+  </define>
+  <define name="RequirementSubpart">
+    <optional>
+      <attribute name="type"/>
+    </optional>
+    <optional>
+      <attribute name="exclude">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="tag"/>
+    </optional>
+    <optional>
+      <attribute name="multilingual-rendering">
+        <ref name="MultilingualRenderingType"/>
+      </attribute>
+    </optional>
+    <oneOrMore>
+      <ref name="BasicBlock"/>
+    </oneOrMore>
+  </define>
+  <define name="ObligationType">
+    <choice>
+      <value>requirement</value>
+      <value>recommendation</value>
+      <value>permission</value>
+    </choice>
+  </define>
+  <define name="classification">
+    <element name="classification">
+      <ref name="classification_tag"/>
+      <ref name="classification_value"/>
+    </element>
+  </define>
+  <define name="classification_tag">
+    <element name="tag">
+      <text/>
+    </element>
+  </define>
+  <define name="classification_value">
+    <element name="value">
+      <text/>
+    </element>
+  </define>
+</grammar>

data/lib/relaton_oasis/data_fetcher.rb

@@ -0,0 +1,91 @@
+module RelatonOasis
+  class DataFetcher
+    #
+    # Initialize a new DataFetcher
+    #
+    # @param [Strin] output directory to save files, default: "data"
+    # @param [Strin] format format of output files (xml, yaml, bibxml); default: yaml
+    #
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = @format.sub(/^bib|^rfc/, "")
+      @files = []
+    end
+
+    #
+    # Initialize fetcher and run fetch
+    #
+    # @param [Strin] output directory to save files, default: "data"
+    # @param [Strin] format format of output files (xml, yaml, bibxml); default: yaml
+    #
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Fetch and save all the documents from OASIS
+    #
+    def fetch
+      agent = Mechanize.new
+      resp = agent.get "https://www.oasis-open.org/standards/"
+      doc = Nokogiri::HTML resp.body
+      doc.xpath("//details").map do |item|
+        save_doc DataParser.new(item).parse
+        fetch_parts item
+      end
+    end
+
+    #
+    # Fetch and save parts of document
+    #
+    # @param [Nokogiri::HTML::Element] item document node
+    #
+    def fetch_parts(item)
+      parts = item.xpath("./div/div/div[contains(@class, 'standard__grid--cite-as')]/p[strong or span/strong]")
+      return unless parts.size > 1
+
+      parts.each do |part|
+        save_doc DataPartParser.new(part).parse
+      end
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonOasis::OasisBibliographicItem] doc 
+    #
+    def save_doc(doc) # rubocop:disable Metrics/MethodLength
+      c = case @format
+          when "xml" then doc.to_xml(bibdata: true)
+          when "yaml" then doc.to_hash.to_yaml
+          else doc.send("to_#{@format}")
+          end
+      file = file_name doc
+      if @files.include? file
+        warn "File #{file} already exists. Document: #{doc.docnumber}"
+      else
+        @files << file
+      end
+      File.write file, c, encoding: "UTF-8"
+    end
+
+    #
+    # Generate file name
+    #
+    # @param [RelatonOasis::OasisBibliographicItem] doc
+    #
+    # @return [String] file name
+    #
+    def file_name(doc)
+      name = doc.docnumber.gsub(/[\s,:\/]/, "_").squeeze("_").upcase
+      File.join @output, "#{name}.#{@ext}"
+    end
+  end
+end

data/lib/relaton_oasis/data_parser.rb

@@ -0,0 +1,204 @@
+module RelatonOasis
+  class DataParser
+    include RelatonOasis::DataParserUtils
+
+    #
+    # Initialize parser.
+    #
+    # @param [Nokogiri::HTML::Element] node docment node
+    #
+    def initialize(node)
+      @node = node
+      @title = @node.at("./summary/div/h2").text
+      @text = @node.at(
+        "./div/div/div[contains(@class, 'standard__grid--cite-as')]/p[em or i or a or span]",
+      )&.text
+    end
+
+    #
+    # Parse document.
+    #
+    # @return [RelatonOasis::OasisBibliographicItem] bibliographic item
+    #
+    def parse # rubocop:disable Metrics/MethodLength
+      RelatonOasis::OasisBibliographicItem.new(
+        fetched: Date.today.to_s,
+        type: "standard",
+        doctype: parse_doctype,
+        title: parse_title,
+        docid: parse_docid,
+        docnumber: parse_docnumber,
+        date: parse_date,
+        abstract: parse_abstract,
+        language: ["en"],
+        script: ["Latn"],
+        editorialgroup: parse_editorialgroup,
+        relation: parse_relation,
+        technology_area: parse_technology_area,
+      )
+    end
+
+    #
+    # Parse title.
+    #
+    # @return [Array<RelatonBib::TypedTitleString>] <description>
+    #
+    def parse_title
+      [RelatonBib::TypedTitleString.new(type: "main", content: @title, language: "en", script: "Latn")]
+    end
+
+    #
+    # Parse date.
+    #
+    # @return [Array<RelatonBib::BibliographicDate>] date
+    #
+    def parse_date
+      @node.xpath("./summary/div/time[@class='standard__date']").map do |d|
+        date_str = d.text.match(/\d{2}\s\w+\s\d{4}/).to_s
+        date = Date.parse(date_str).to_s
+        RelatonBib::BibliographicDate.new(on: date, type: "issued")
+      end
+    end
+
+    #
+    # Parse abstract.
+    #
+    # @return [Array<RelatonBib::FormattedString>] abstract
+    #
+    def parse_abstract
+      c = @node.xpath(
+        "./summary/div/div[@class='standard__description']/p",
+      ).map { |a| a.text.gsub(/[\n\t]+/, " ").strip }.join("\n")
+      return [] if c.empty?
+
+      [RelatonBib::FormattedString.new(content: c, language: "en", script: "Latn")]
+    end
+
+    #
+    # Parse technical committee.
+    #
+    # @return [RelatonBib::EditorialGroup] technical committee
+    #
+    def parse_editorialgroup
+      tc = @node.xpath("./div[@class='standard__details']/a").map do |a|
+        wg = RelatonBib::WorkGroup.new name: a.text.strip
+        RelatonBib::TechnicalCommittee.new wg
+      end
+      RelatonBib::EditorialGroup.new tc
+    end
+
+    #
+    # Parse relation.
+    #
+    # @return [Array<RelatonBib::DocumentRelation>] relation
+    #
+    def parse_relation
+      rels = @node.xpath(
+        "./div/div/div[contains(@class, 'standard__grid--cite-as')]/p[strong or span/strong or b/span]",
+      )
+      return [] unless rels.size > 1
+
+      rels.map do |r|
+        docid = DataPartParser.new(r).parse_docid
+        fref = RelatonBib::FormattedRef.new content: docid[0].id
+        bib = RelatonOasis::OasisBibliographicItem.new formattedref: fref
+        RelatonBib::DocumentRelation.new type: "hasPart", bibitem: bib
+      end
+    end
+
+    #
+    # Parse document part references.
+    #
+    # @return [Array<String>] document part references
+    #
+    def document_part_refs
+      @node.css(
+        ".standard__grid--cite-as > p > strong",
+        "span.Refterm", "span.abbrev", "span.citationLabel > strong"
+      ).map { |p| p.text.gsub(/^\[{1,2}|\]$/, "").strip }
+    end
+
+    #
+    # Parse document number.
+    #
+    # @return [String] document number
+    #
+    def parse_docnumber
+      parts = document_part_refs
+      case parts.size
+      when 0 then title_to_docid @node.at("./summary/div/h2").text
+      when 1 then parse_spec(parts[0])
+      else parts_to_docid parts
+      end
+    end
+
+    #
+    # Create document identifier from parts references.
+    #
+    # @param [Array<String>] parts parts references
+    #
+    # @return [String] document identifier
+    #
+    def parts_to_docid(parts)
+      id = parts[1..-1].each_with_object(parts[0].split("-")) do |part, acc|
+        chunks = part.split "-"
+        chunks.each.with_index do |chunk, idx|
+          unless chunk.casecmp(acc[idx])&.zero?
+            acc.slice!(idx..-1)
+            break
+          end
+        end
+      end.join("-")
+      parse_spec(id)
+    end
+
+    #
+    # Create document identifier from title.
+    #
+    # @param [String] title title
+    #
+    # @return [String] document identifier
+    #
+    def title_to_docid(title) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      abbrs = title.scan(/(?<=\()[^)]+(?=\))/)
+      if abbrs.any?
+        id = abbrs.map { |abbr| abbr.split.join("-") }.join "-"
+        /(?:Version\s|v)(?<ver>[\d.]+)/ =~ title
+        id += "-v#{ver}" if ver
+        /(?<eb>ebXML|ebMS)/ =~ title
+        id = "#{eb}-#{id}" if eb
+        id
+      else
+        series_end = false
+        title.sub(/\s\[OASIS\s\d+\]$/, "").split(/[,:]?\s|-|(?<=[a-z])(?=[A-Z][a-z])/)
+          .each_with_object([""]) do |word, acc|
+          if word =~ /^v[\d.]+/
+            acc << $MATCH.to_s
+            series_end = true
+          elsif word.match?(/^Version/)
+            acc << "v"
+            series_end = false
+          elsif word.match?(/^\d|ebXML|ebMS/)
+            series_end ? acc << word : acc[-1] += word
+            series_end = true
+          elsif word.match?(/^\w+$/) && word == word.upcase
+            series_end ? acc << word : acc[-1] = word
+            series_end = true
+          elsif word.match?(/[A-Z]+[a-z]+/)
+            series_end ? acc << word[0] : acc[-1] += word[0]
+            series_end = false
+          end
+        end.join "-"
+      end
+    end
+
+    #
+    # Parse technology areas.
+    #
+    # @return [Array<String>] technology areas
+    #
+    def parse_technology_area
+      super @node
+    end
+  end
+end

data/lib/relaton_oasis/data_parser_utils.rb

@@ -0,0 +1,89 @@
+module RelatonOasis
+  module DataParserUtils
+    #
+    # Parse document identifier specification.
+    #
+    # @param [String] num document number
+    #
+    # @return [String] document identifier with specification if needed
+    #
+    def parse_spec(num)
+      id = case @text
+           when /OASIS Project Specification (\d+)/ then "#{num}-PS#{$1}"
+           when /Committee Specification (\d+)/ then "#{num}-CS#{$1}"
+           else num
+           end
+      parse_part(id)
+    end
+
+    #
+    # Parse document identifier part.
+    #
+    # @param [<Type>] docid <description>
+    #
+    # @return [<Type>] <description>
+    #
+    def parse_part(docid)
+      return docid if docid.match?(/(?:Part|Pt)\d+/i)
+
+      id = case @title
+           when /Part\s(\d+)/ then "#{docid}-Pt#{$1}"
+           else docid
+           end
+      parse_errata(id)
+    end
+
+    #
+    # Parse document identifier errata.
+    #
+    # @param [String] id document identifier
+    #
+    # @return [String] document identifier with errata if needed
+    #
+    def parse_errata(id)
+      return id if id.match?(/errata\d+/i)
+
+      case @title
+      when /Plus\sErrata\s(\d+)/ then "#{id}-plus-errata#{$1}"
+      when /Errata\s(\d+)/ then "#{id}-errata#{$1}"
+      else id
+      end
+    end
+
+    #
+    # Parse document identifier.
+    #
+    # @return [Array<RelatonBib::DocumentIdentifier>] document identifier
+    #
+    def parse_docid
+      id = "OASIS #{parse_docnumber}"
+      [RelatonBib::DocumentIdentifier.new(type: "OASIS", id: id, primary: true)]
+    end
+
+    #
+    # Parse document type.
+    #
+    # @return [String] document type
+    #
+    def parse_doctype
+      case @text
+      when /OASIS Project Specification/, /Committee Specification/
+        "specification"
+      when /Technical Memorandum/ then "memorandum"
+      when /Technical Resolution/ then "resolution"
+      else "standard"
+      end
+    end
+
+    #
+    # Parse technology area.
+    #
+    # @return [Array<String>] technology areas
+    #
+    def parse_technology_area(node)
+      node.xpath(
+        "./summary/div/div/ul[@class='technology-areas__list']/li/a",
+      ).map { |ta| ta.text.strip.gsub(/\s/, "-") }
+    end
+  end
+end