relaton-ietf 1.8.0 → 1.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5f2fabf76d43dd9c8ef97a422edb9c3c9123d74b47a3cd8ea8fd440db589d74
-  data.tar.gz: 26abc7fd8b1f1077274fa0c8244dbf38e69722efdb6597b6939aeaa76556e6fe
+  metadata.gz: 05a6c063cec7b11a826ebe390cc8ebc70974504a70098886a9ac815036428314
+  data.tar.gz: e99a83cfdc93c51e58af1c50baba6a4b518a1895da6a7949cb4fa2e500b56f0b
 SHA512:
-  metadata.gz: 81ff3b89ff6b4d69cb5bcb2b42a5a767c0ee96d8a6e18c6c5defbb268f1148cdd5d806a951a680ef08b6e9cf4848df21fe5acb95fc2b7593c88273d850960c43
-  data.tar.gz: '0698205d63e5e996b836bdf82d1f7128da9c45f8c880adb18cd14254293e934d75ec633c4363afa3d09d2bae12a9448417300475e44aca31c33ddce1166ef35b'
+  metadata.gz: 10ec58d5177d45a71079daee30517c3ea040cf5b231ff5fb0688c10818ba7d77b0cd5e61cec478739851a1a460d4b37cec8dcb1d4ce3b53ae2a84bca3300348f
+  data.tar.gz: 4fda1bcdab90ddc307f11d9684016b625d2f7c7221aeb25b9f7fcc1b1679f65b2829833fd982df913e8a7f0dfc910522bf2905dd7555c8da30e80cfa64ae18fb

data/.github/workflows/rake.yml

@@ -16,19 +16,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
+        ruby: ['3.0', '2.7', '2.6', '2.5' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
-        include:
-          - ruby: '3.0'
-            os: 'ubuntu-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'windows-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'macos-latest'
-            experimental: true
     steps:
       - uses: actions/checkout@v2
         with:

data/.rubocop.yml

@@ -2,9 +2,11 @@
 # https://github.com/riboseinc/oss-guides
 # All project-specific additions and overrides should be specified in this file.
 
+require: rubocop-rails
+
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5
 Rails:
   Enabled: false

data/README.adoc

@@ -90,6 +90,21 @@ item.to_xml bibdata: true
 </bibdata>"
 ----
 
+=== Typed links
+
+IETF documents may have `src`, `xml`, and `doi` link type.
+
+* `src` - web pulication
+* `xml` - BibXML publication
+* `doi` - DOI reference
+
+[source,ruby]
+----
+item.link
+=> [#<RelatonBib::TypedUri:0x00007fe8b287a120 @content=#<Addressable::URI:0x7e4 URI:https://raw.githubusercontent.com/relaton/relaton-data-ietf/master/data/reference.RFC.8341.xml>, @type="xml">,
+ #<RelatonBib::TypedUri:0x00007fe8b2237ec0 @content=#<Addressable::URI:0x7f8 URI:https://www.rfc-editor.org/info/rfc8341>, @type="src">]
+----
+
 === Parse a file locally
 
 [source,ruby]
@@ -118,6 +133,41 @@ RelatonIetf::IetfBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+There is a IETF datasets what can be converted into RelatonXML/BibXML/BibYAML formats:
+
+- `ietf-rfcsubseries` - https://www.rfc-editor.org/rfc-index.xml (`<bcp-entry>`, `<fyi-entry>`, `<std-entry>`)
+- `ietf-internet-drafts` - https://www.ietf.org/lib/dt/sprint/bibxml-ids.tgz
+- `ietf-rfc-entries` - https://www.rfc-editor.org/rfc-index.xml (`<rfc-entry>`)
+
+The method `RelatonIetf::DataFetcher.fetch(source, output: "data", format: "yaml")` converts all the documents from the dataset and save them to the `./data` folder in YAML format.
+
+Arguments:
+
+- `source` - dataset name (`ietf-rfcsubseries` or `ietf-internet-drafts`)
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml`, `bibxml` (default `yaml`).
+
+For `ietf-rfcsubseries` dataset only special XML format is supported:
+
+[sourse.xml]
+----
+<referencegroup anchor="BCP14" target="https://www.rfc-editor.org/info/bcp14">
+  <xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.2119.xml" />
+  <xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.8174.xml" />
+</referencegroup>
+----
+
+[source,ruby]
+----
+RelatonIetf::DataFetcher.fetch "ietf-internet-drafts"
+Started at: 2021-12-17 10:23:20 +0100
+Stopped at: 2021-12-17 10:29:19 +0100
+Done in: 360 sec.
+=> nil
+----
+
 == Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/metanorma/relaton-ietf.

data/grammars/biblio.rng

@@ -787,6 +787,7 @@
       <value>adapted</value>
       <value>vote-started</value>
       <value>vote-ended</value>
+      <value>announced</value>
     </choice>
   </define>
   <define name="bdate">

data/grammars/ietf.rng

@@ -19,6 +19,9 @@
       <optional>
         <ref name="doctype"/>
       </optional>
+      <optional>
+        <ref name="docsubtype"/>
+      </optional>
       <zeroOrMore>
         <ref name="editorialgroup"/>
       </zeroOrMore>

data/grammars/isodoc.rng

@@ -45,6 +45,11 @@
         <optional>
           <attribute name="alt"/>
         </optional>
+        <optional>
+          <attribute name="updatetype">
+            <data type="boolean"/>
+          </attribute>
+        </optional>
         <text/>
       </element>
     </define>
@@ -199,6 +204,18 @@
         </zeroOrMore>
       </element>
     </define>
+    <define name="dt">
+      <element name="dt">
+        <optional>
+          <attribute name="id">
+            <data type="ID"/>
+          </attribute>
+        </optional>
+        <zeroOrMore>
+          <ref name="TextElement"/>
+        </zeroOrMore>
+      </element>
+    </define>
     <define name="example">
       <element name="example">
         <attribute name="id">
@@ -543,6 +560,9 @@
     </define>
     <define name="BibDataExtensionType">
       <ref name="doctype"/>
+      <optional>
+        <ref name="docsubtype"/>
+      </optional>
       <optional>
         <ref name="editorialgroup"/>
       </optional>
@@ -890,6 +910,14 @@
     </define>
   </include>
   <!-- end overrides -->
+  <define name="docsubtype">
+    <element name="subdoctype">
+      <ref name="DocumentSubtype"/>
+    </element>
+  </define>
+  <define name="DocumentSubtype">
+    <text/>
+  </define>
   <define name="colgroup">
     <element name="colgroup">
       <oneOrMore>
@@ -939,7 +967,34 @@
   <define name="concept">
     <element name="concept">
       <optional>
-        <attribute name="term"/>
+        <attribute name="ital">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="ref">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
+      <optional>
+        <element name="refterm">
+          <zeroOrMore>
+            <choice>
+              <ref name="PureTextElement"/>
+              <ref name="stem"/>
+            </choice>
+          </zeroOrMore>
+        </element>
+      </optional>
+      <optional>
+        <element name="renderterm">
+          <zeroOrMore>
+            <choice>
+              <ref name="PureTextElement"/>
+              <ref name="stem"/>
+            </choice>
+          </zeroOrMore>
+        </element>
       </optional>
       <choice>
         <ref name="eref"/>
@@ -965,6 +1020,9 @@
       </attribute>
       <attribute name="name"/>
       <attribute name="action"/>
+      <optional>
+        <attribute name="class"/>
+      </optional>
       <zeroOrMore>
         <choice>
           <ref name="TextElement"/>
@@ -1191,13 +1249,17 @@
   </define>
   <define name="IsoWorkgroup">
     <optional>
-      <attribute name="number">
-        <data type="int"/>
-      </attribute>
+      <attribute name="number"/>
     </optional>
     <optional>
       <attribute name="type"/>
     </optional>
+    <optional>
+      <attribute name="identifier"/>
+    </optional>
+    <optional>
+      <attribute name="prefix"/>
+    </optional>
     <text/>
   </define>
   <define name="ics">
@@ -1459,26 +1521,26 @@
     <optional>
       <ref name="section-title"/>
     </optional>
-    <group>
+    <choice>
       <choice>
         <group>
-          <zeroOrMore>
+          <oneOrMore>
             <ref name="BasicBlock"/>
-          </zeroOrMore>
+          </oneOrMore>
           <zeroOrMore>
             <ref name="note"/>
           </zeroOrMore>
         </group>
         <ref name="amend"/>
       </choice>
-      <zeroOrMore>
+      <oneOrMore>
         <choice>
           <ref name="clause-subsection"/>
           <ref name="terms"/>
           <ref name="definitions"/>
         </choice>
-      </zeroOrMore>
-    </group>
+      </oneOrMore>
+    </choice>
   </define>
   <define name="Annex-Section">
     <optional>

data/lib/relaton_ietf/committee.rb

@@ -0,0 +1,8 @@
+module RelatonIetf
+  class Committee < RelatonBib::TechnicalCommittee
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(builder)
+      builder.committee { |b| workgroup.to_xml b }
+    end
+  end
+end

data/lib/relaton_ietf/data_fetcher.rb

@@ -0,0 +1,130 @@
+require "rubygems"
+require "rubygems/package"
+require "zlib"
+require "relaton_ietf/rfc_index_entry"
+require "relaton_ietf/rfc_entry"
+
+module RelatonIetf
+  class DataFetcher
+    #
+    # Data fetcher initializer
+    #
+    # @param [String] source source name
+    # @param [String] output directory to save files
+    # @param [String] format format of output files (xml, yaml, bibxml);
+    #   for ietf-rfcsubseries source only: xml
+    #
+    def initialize(source, output, format)
+      @source = source
+      @output = output
+      @format = source == "ietf-rfcsubseries" ? "rfcxml" : format
+      @ext = @format.sub(/^bib|^rfc/, "")
+      @files = []
+    end
+
+    #
+    # Initialize fetcher and run fetch
+    #
+    # @param [String] source source name
+    # @param [Strin] output directory to save files, default: "data"
+    # @param [Strin] format format of output files (xml, yaml, bibxml);
+    #   default: yaml; for ietf-rfcsubseries source only: xml
+    #
+    def self.fetch(source, output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(source, output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Fetch documents
+    #
+    def fetch
+      case @source
+      when "ietf-rfcsubseries" then fetch_ieft_rfcsubseries
+      when "ietf-internet-drafts" then fetch_ieft_internet_drafts
+      when "ietf-rfc-entries" then fetch_ieft_rfcs
+      end
+    end
+
+    #
+    # Fetches ietf-rfcsubseries documents
+    #
+    def fetch_ieft_rfcsubseries
+      rfc_index.xpath("xmlns:bcp-entry|xmlns:fyi-entry|xmlns:std-entry").each do |doc|
+        save_doc RfcIndexEntry.parse(doc)
+      end
+    end
+
+    #
+    # Fetches ietf-internet-drafts documents
+    #
+    def fetch_ieft_internet_drafts # rubocop:disable Metrics/MethodLength
+      gz = OpenURI.open_uri("https://www.ietf.org/lib/dt/sprint/bibxml-ids.tgz")
+      z = Zlib::GzipReader.new(gz)
+      io = StringIO.new(z.read)
+      z.close
+      Gem::Package::TarReader.new io do |tar|
+        tar.each do |tarfile|
+          next if tarfile.directory?
+
+          save_doc RelatonBib::BibXMLParser.parse(tarfile.read)
+        end
+      end
+    end
+
+    def fetch_ieft_rfcs
+      rfc_index.xpath("xmlns:rfc-entry").each do |doc|
+        save_doc RfcEntry.parse(doc)
+      end
+    end
+
+    def rfc_index
+      uri = URI "https://www.rfc-editor.org/rfc-index.xml"
+      Nokogiri::XML(Net::HTTP.get(uri)).at("/xmlns:rfc-index")
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonIetf::RfcIndexEntry, nil] rfc index entry
+    #
+    def save_doc(entry) # rubocop:disable Metrics/MethodLength
+      return unless entry
+
+      c = case @format
+          when "xml" then entry.to_xml(bibdata: true)
+          when "yaml" then entry.to_hash.to_yaml
+          when "rfcxml" then entry.to_xml
+          else entry.send("to_#{@format}")
+          end
+      file = file_name entry
+      if @files.include? file
+        warn "File #{file} already exists. Document: #{entry.docnumber}"
+      else
+        @files << file
+      end
+      File.write file, c, encoding: "UTF-8"
+    end
+
+    #
+    # Generate file name
+    #
+    # @param [RelatonIetf::RfcIndexEntry] entry
+    #
+    # @return [String] file name
+    #
+    def file_name(entry)
+      id = if entry.respond_to? :docidentifier
+             entry.docidentifier.detect { |i| i.type == "Internet-Draft" }&.id
+           end
+      id ||= entry.docnumber
+      name = id.gsub(/[\s,:\/]/, "_").squeeze("_").upcase
+      File.join @output, "#{name}.#{@ext}"
+    end
+  end
+end

data/lib/relaton_ietf/ietf_bibliographic_item.rb

@@ -21,7 +21,7 @@ module RelatonIetf
     # @return [RelatonIetf::IetfBibliographicItem]
     def self.from_hash(hash)
       item_hash = ::RelatonIetf::HashConverter.hash_to_bib(hash)
-      new **item_hash
+      new(**item_hash)
     end
 
     # @param opts [Hash]
@@ -32,7 +32,7 @@ module RelatonIetf
     # @return [String] XML
     def to_xml(**opts)
       opts[:date_format] ||= :short
-      super **opts
+      super(**opts)
     end
   end
 end

data/lib/relaton_ietf/processor.rb

@@ -3,11 +3,12 @@ require "relaton_ietf/xml_parser"
 
 module RelatonIetf
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_ietf
       @prefix = "IETF"
       @defaultprefix = /^RFC /
       @idtype = "IETF"
+      @datasets = %w[ietf-rfcsubseries ietf-internet-drafts ietf-rfc-entries]
     end
 
     # @param code [String]
@@ -18,6 +19,18 @@ module RelatonIetf
       ::RelatonIetf::IetfBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from https://www.rfc-editor.org/rfc-index.xml
+    #
+    # @param [String] source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(source, opts)
+      DataFetcher.fetch(source, **opts)
+    end
+
     # @param xml [String]
     # @return [RelatonIetf::IetfBibliographicItem]
     def from_xml(xml)

data/lib/relaton_ietf/rfc_entry.rb

@@ -0,0 +1,186 @@
+module RelatonIetf
+  class RfcEntry
+    #
+    # Initalize parser
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    def initialize(doc)
+      @doc = doc
+    end
+
+    #
+    # Initialize parser & parse document
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    # @return [RelatonIetf::IetfBibliographicItem] bib item
+    #
+    def self.parse(doc)
+      new(doc).parse
+    end
+
+    #
+    # Parse document
+    #
+    # @return [RelatonIetf::IetfBibliographicItem] bib item
+    #
+    def parse # rubocop:disable Metrics/MethodLength
+      IetfBibliographicItem.new(
+        type: "standard",
+        language: ["en"],
+        script: ["Latn"],
+        fetched: Date.today.to_s,
+        docid: parse_docid,
+        docnumber: code,
+        title: parse_title,
+        link: parse_link,
+        date: parse_date,
+        contributor: parse_contributor,
+        keyword: parse_keyword,
+        abstract: parse_abstract,
+        relation: parse_relation,
+        status: parse_status,
+        editorialgroup: parse_editorialgroup,
+      )
+    end
+
+    #
+    # Parse document identifiers
+    #
+    # @return [Array<RelatonBib::DocumentIdettifier>] document identifiers
+    #
+    def parse_docid
+      ids = [RelatonBib::DocumentIdentifier.new(id: pub_id, type: "IETF")]
+      doi = @doc.at("./xmlns:doi").text
+      ids << RelatonBib::DocumentIdentifier.new(id: doi, type: "DOI")
+      ids
+    end
+
+    #
+    # Parse document title
+    #
+    # @return [Array<RelatonBib::TypedTileString>] document title
+    #
+    def parse_title
+      content = @doc.at("./xmlns:title").text
+      [RelatonBib::TypedTitleString.new(content: content, type: "main")]
+    end
+
+    #
+    # Create PubID
+    #
+    # @return [String] PubID
+    #
+    def pub_id
+      "IETF #{code}"
+    end
+
+    #
+    # Parse document code
+    #
+    # @return [String] document code
+    #
+    def code
+      @doc.at("./xmlns:doc-id").text
+    end
+
+    #
+    # Create link
+    #
+    # @return [Array<RelatonBib::TypedUri>] 
+    #
+    def parse_link
+      num = code[-4..-1].sub(/^0+/, "")
+      url = "https://www.rfc-editor.org/info/rfc#{num}"
+      [RelatonBib::TypedUri.new(content: url, type: "src")]
+    end
+
+    #
+    # Parse document date
+    #
+    # @return [Array<RelatonBib::BibliographicDate>] document date
+    #
+    def parse_date
+      @doc.xpath("./xmlns:date").map do |date|
+        month = date.at("./xmlns:month").text
+        year = date.at("./xmlns:year").text
+        on = "#{year}-#{Date::MONTHNAMES.index(month).to_s.rjust(2, '0')}"
+        RelatonBib::BibliographicDate.new(on: on, type: "published")
+      end
+    end
+
+    #
+    # Parse document contributors
+    #
+    # @return [Array<RelatonBib::ContributionInfo>] document contributors
+    #
+    def parse_contributor
+      @doc.xpath("./xmlns:author").map do |contributor|
+        n = contributor.at("./xmlns:name").text
+        name = RelatonBib::LocalizedString.new( n,  "en",  "Latn")
+        fname = RelatonBib::FullName.new(completename: name)
+        person = RelatonBib::Person.new(name: fname)
+        RelatonBib::ContributionInfo.new(entity: person, role: [{ type: "author" }])
+      end
+    end
+
+    #
+    # Parse document keywords
+    #
+    # @return [Array<String>] document keywords
+    #
+    def parse_keyword
+      @doc.xpath("./xmlns:keywords/xmlns:kw").map &:text
+    end
+
+    #
+    # Parse document abstract
+    #
+    # @return [Array<RelatonBib::FormattedString>] document abstract
+    #
+    def parse_abstract
+      @doc.xpath("./xmlns:abstract").map do |c|
+        RelatonBib::FormattedString.new(content: c.text, language: "en",
+                                        script: "Latn", format: "text/html")
+      end
+    end
+
+    #
+    # Parse document relations
+    #
+    # @return [Arra<RelatonBib::DocumentRelation>] document relations
+    #
+    def parse_relation
+      types = { "updates" => "updates", "obsoleted-by" => "obsoletedBy"}
+      @doc.xpath("./xmlns:updates/xmlns:doc-id|./xmlns:obsoleted-by/xmlns:doc-id").map do |r|
+        fref = RelatonBib::FormattedRef.new(content: r.text)
+        bib = IetfBibliographicItem.new(formattedref: fref)
+        RelatonBib::DocumentRelation.new(type: types[r.parent.name], bibitem: bib)
+      end
+    end
+
+    #
+    # Parse document status
+    #
+    # @return [RelatonBib::DocuemntStatus] document status
+    #
+    def parse_status
+      stage = @doc.at("./xmlns:current-status").text
+      RelatonBib::DocumentStatus.new(stage: stage)
+    end
+
+    #
+    # Parse document editorial group
+    #
+    # @return [RelatonBib::EditorialGroup] document editorial group
+    #
+    def parse_editorialgroup
+      tc = @doc.xpath("./xmlns:wg_acronym").map do |wg|
+        wg = RelatonBib::WorkGroup.new(name: wg.text)
+        RelatonBib::TechnicalCommittee.new(wg)
+      end
+      RelatonBib::EditorialGroup.new(tc)
+    end
+  end
+end