relaton-ietf 1.9.0 → 1.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de34f43de3864f7494ab3931162ecff103d9f2f66828374bc26ccc91da3c3295
-  data.tar.gz: 84f76c4b573041e55f60efdf208480c6832129a9a094bc3c12205bc19ba42552
+  metadata.gz: 639209bd0a93d43f7d9d671d709a40e635ef7978ac33b502afde91b64e03c857
+  data.tar.gz: d33475e9becda05d7eadc6b9e48e3df700aa4620dc6081811708faddd84b650f
 SHA512:
-  metadata.gz: d6f0c509201f46a402655f2ea4d75b678cb6c4638715a07cd637c8b5c8fa3dc04254ea914dfcd27eebe024231b8caec5b2435f944bc5d8185622b1b515dff3ab
-  data.tar.gz: bd93874e76dad53e998abd0988aba64354a7a5d011141f57e945bb2a2e7c690e0b790c58a343070160bee77d2e8cb7332fe5b2a59d2c983b3222e9bc55db09f7
+  metadata.gz: 87f20d0e95cc16aa9462b78cf67eef1ca40e0124bda85ec2a480de960b44d33ef333083ce25fdd2fff6e109b071fdbf2cf27226c210eef65331b1786daa59f85
+  data.tar.gz: 3865277ac823c1fe10731a813ca5085d9e3ced27bb5af6e1bb53cb9a7c952c76e47f249cbb6cc013293c5ba2f46410f309689be307e55e1fc1aff4d508219d88

data/.rubocop.yml

@@ -2,6 +2,8 @@
 # https://github.com/riboseinc/oss-guides
 # All project-specific additions and overrides should be specified in this file.
 
+require: rubocop-rails
+
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:

data/README.adoc

@@ -90,6 +90,21 @@ item.to_xml bibdata: true
 </bibdata>"
 ----
 
+=== Typed links
+
+IETF documents may have `src`, `xml`, and `doi` link type.
+
+* `src` - web pulication
+* `xml` - BibXML publication
+* `doi` - DOI reference
+
+[source,ruby]
+----
+item.link
+=> [#<RelatonBib::TypedUri:0x00007fe8b287a120 @content=#<Addressable::URI:0x7e4 URI:https://raw.githubusercontent.com/relaton/relaton-data-ietf/master/data/reference.RFC.8341.xml>, @type="xml">,
+ #<RelatonBib::TypedUri:0x00007fe8b2237ec0 @content=#<Addressable::URI:0x7f8 URI:https://www.rfc-editor.org/info/rfc8341>, @type="src">]
+----
+
 === Parse a file locally
 
 [source,ruby]
@@ -118,6 +133,41 @@ RelatonIetf::IetfBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+There is a IETF datasets what can be converted into RelatonXML/BibXML/BibYAML formats:
+
+- `ietf-rfcsubseries` - https://www.rfc-editor.org/rfc-index.xml (`<bcp-entry>`, `<fyi-entry>`, `<std-entry>`)
+- `ietf-internet-drafts` - https://www.ietf.org/lib/dt/sprint/bibxml-ids.tgz
+- `ietf-rfc-entries` - https://www.rfc-editor.org/rfc-index.xml (`<rfc-entry>`)
+
+The method `RelatonIetf::DataFetcher.fetch(source, output: "data", format: "yaml")` converts all the documents from the dataset and save them to the `./data` folder in YAML format.
+
+Arguments:
+
+- `source` - dataset name (`ietf-rfcsubseries` or `ietf-internet-drafts`)
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml`, `bibxml` (default `yaml`).
+
+For `ietf-rfcsubseries` dataset only special XML format is supported:
+
+[sourse.xml]
+----
+<referencegroup anchor="BCP14" target="https://www.rfc-editor.org/info/bcp14">
+  <xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.2119.xml" />
+  <xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.8174.xml" />
+</referencegroup>
+----
+
+[source,ruby]
+----
+RelatonIetf::DataFetcher.fetch "ietf-internet-drafts"
+Started at: 2021-12-17 10:23:20 +0100
+Stopped at: 2021-12-17 10:29:19 +0100
+Done in: 360 sec.
+=> nil
+----
+
 == Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/metanorma/relaton-ietf.

data/lib/relaton_ietf/committee.rb

@@ -0,0 +1,8 @@
+module RelatonIetf
+  class Committee < RelatonBib::TechnicalCommittee
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(builder)
+      builder.committee { |b| workgroup.to_xml b }
+    end
+  end
+end

data/lib/relaton_ietf/data_fetcher.rb

@@ -0,0 +1,130 @@
+require "rubygems"
+require "rubygems/package"
+require "zlib"
+require "relaton_ietf/rfc_index_entry"
+require "relaton_ietf/rfc_entry"
+
+module RelatonIetf
+  class DataFetcher
+    #
+    # Data fetcher initializer
+    #
+    # @param [String] source source name
+    # @param [String] output directory to save files
+    # @param [String] format format of output files (xml, yaml, bibxml);
+    #   for ietf-rfcsubseries source only: xml
+    #
+    def initialize(source, output, format)
+      @source = source
+      @output = output
+      @format = source == "ietf-rfcsubseries" ? "rfcxml" : format
+      @ext = @format.sub(/^bib|^rfc/, "")
+      @files = []
+    end
+
+    #
+    # Initialize fetcher and run fetch
+    #
+    # @param [String] source source name
+    # @param [Strin] output directory to save files, default: "data"
+    # @param [Strin] format format of output files (xml, yaml, bibxml);
+    #   default: yaml; for ietf-rfcsubseries source only: xml
+    #
+    def self.fetch(source, output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(source, output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Fetch documents
+    #
+    def fetch
+      case @source
+      when "ietf-rfcsubseries" then fetch_ieft_rfcsubseries
+      when "ietf-internet-drafts" then fetch_ieft_internet_drafts
+      when "ietf-rfc-entries" then fetch_ieft_rfcs
+      end
+    end
+
+    #
+    # Fetches ietf-rfcsubseries documents
+    #
+    def fetch_ieft_rfcsubseries
+      rfc_index.xpath("xmlns:bcp-entry|xmlns:fyi-entry|xmlns:std-entry").each do |doc|
+        save_doc RfcIndexEntry.parse(doc)
+      end
+    end
+
+    #
+    # Fetches ietf-internet-drafts documents
+    #
+    def fetch_ieft_internet_drafts # rubocop:disable Metrics/MethodLength
+      gz = OpenURI.open_uri("https://www.ietf.org/lib/dt/sprint/bibxml-ids.tgz")
+      z = Zlib::GzipReader.new(gz)
+      io = StringIO.new(z.read)
+      z.close
+      Gem::Package::TarReader.new io do |tar|
+        tar.each do |tarfile|
+          next if tarfile.directory?
+
+          save_doc RelatonBib::BibXMLParser.parse(tarfile.read)
+        end
+      end
+    end
+
+    def fetch_ieft_rfcs
+      rfc_index.xpath("xmlns:rfc-entry").each do |doc|
+        save_doc RfcEntry.parse(doc)
+      end
+    end
+
+    def rfc_index
+      uri = URI "https://www.rfc-editor.org/rfc-index.xml"
+      Nokogiri::XML(Net::HTTP.get(uri)).at("/xmlns:rfc-index")
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonIetf::RfcIndexEntry, nil] rfc index entry
+    #
+    def save_doc(entry) # rubocop:disable Metrics/MethodLength
+      return unless entry
+
+      c = case @format
+          when "xml" then entry.to_xml(bibdata: true)
+          when "yaml" then entry.to_hash.to_yaml
+          when "rfcxml" then entry.to_xml
+          else entry.send("to_#{@format}")
+          end
+      file = file_name entry
+      if @files.include? file
+        warn "File #{file} already exists. Document: #{entry.docnumber}"
+      else
+        @files << file
+      end
+      File.write file, c, encoding: "UTF-8"
+    end
+
+    #
+    # Generate file name
+    #
+    # @param [RelatonIetf::RfcIndexEntry] entry
+    #
+    # @return [String] file name
+    #
+    def file_name(entry)
+      id = if entry.respond_to? :docidentifier
+             entry.docidentifier.detect { |i| i.type == "Internet-Draft" }&.id&.downcase
+           end
+      id ||= entry.docnumber.upcase
+      name = id.gsub(/[\s,:\/]/, "_").squeeze("_")
+      File.join @output, "#{name}.#{@ext}"
+    end
+  end
+end

data/lib/relaton_ietf/ietf_bibliographic_item.rb

@@ -21,7 +21,7 @@ module RelatonIetf
     # @return [RelatonIetf::IetfBibliographicItem]
     def self.from_hash(hash)
       item_hash = ::RelatonIetf::HashConverter.hash_to_bib(hash)
-      new **item_hash
+      new(**item_hash)
     end
 
     # @param opts [Hash]
@@ -32,7 +32,7 @@ module RelatonIetf
     # @return [String] XML
     def to_xml(**opts)
       opts[:date_format] ||= :short
-      super **opts
+      super(**opts)
     end
   end
 end

data/lib/relaton_ietf/processor.rb

@@ -3,11 +3,12 @@ require "relaton_ietf/xml_parser"
 
 module RelatonIetf
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_ietf
       @prefix = "IETF"
       @defaultprefix = /^RFC /
       @idtype = "IETF"
+      @datasets = %w[ietf-rfcsubseries ietf-internet-drafts ietf-rfc-entries]
     end
 
     # @param code [String]
@@ -18,6 +19,18 @@ module RelatonIetf
       ::RelatonIetf::IetfBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from https://www.rfc-editor.org/rfc-index.xml
+    #
+    # @param [String] source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(source, opts)
+      DataFetcher.fetch(source, **opts)
+    end
+
     # @param xml [String]
     # @return [RelatonIetf::IetfBibliographicItem]
     def from_xml(xml)

data/lib/relaton_ietf/rfc_entry.rb

@@ -0,0 +1,186 @@
+module RelatonIetf
+  class RfcEntry
+    #
+    # Initalize parser
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    def initialize(doc)
+      @doc = doc
+    end
+
+    #
+    # Initialize parser & parse document
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    # @return [RelatonIetf::IetfBibliographicItem] bib item
+    #
+    def self.parse(doc)
+      new(doc).parse
+    end
+
+    #
+    # Parse document
+    #
+    # @return [RelatonIetf::IetfBibliographicItem] bib item
+    #
+    def parse # rubocop:disable Metrics/MethodLength
+      IetfBibliographicItem.new(
+        type: "standard",
+        language: ["en"],
+        script: ["Latn"],
+        fetched: Date.today.to_s,
+        docid: parse_docid,
+        docnumber: code,
+        title: parse_title,
+        link: parse_link,
+        date: parse_date,
+        contributor: parse_contributor,
+        keyword: parse_keyword,
+        abstract: parse_abstract,
+        relation: parse_relation,
+        status: parse_status,
+        editorialgroup: parse_editorialgroup,
+      )
+    end
+
+    #
+    # Parse document identifiers
+    #
+    # @return [Array<RelatonBib::DocumentIdettifier>] document identifiers
+    #
+    def parse_docid
+      ids = [RelatonBib::DocumentIdentifier.new(id: pub_id, type: "IETF")]
+      doi = @doc.at("./xmlns:doi").text
+      ids << RelatonBib::DocumentIdentifier.new(id: doi, type: "DOI")
+      ids
+    end
+
+    #
+    # Parse document title
+    #
+    # @return [Array<RelatonBib::TypedTileString>] document title
+    #
+    def parse_title
+      content = @doc.at("./xmlns:title").text
+      [RelatonBib::TypedTitleString.new(content: content, type: "main")]
+    end
+
+    #
+    # Create PubID
+    #
+    # @return [String] PubID
+    #
+    def pub_id
+      "IETF #{code.sub(/^(RFC)(\d+)/, '\1 \2')}"
+    end
+
+    #
+    # Parse document code
+    #
+    # @return [String] document code
+    #
+    def code
+      @doc.at("./xmlns:doc-id").text
+    end
+
+    #
+    # Create link
+    #
+    # @return [Array<RelatonBib::TypedUri>] 
+    #
+    def parse_link
+      num = code[-4..-1].sub(/^0+/, "")
+      url = "https://www.rfc-editor.org/info/rfc#{num}"
+      [RelatonBib::TypedUri.new(content: url, type: "src")]
+    end
+
+    #
+    # Parse document date
+    #
+    # @return [Array<RelatonBib::BibliographicDate>] document date
+    #
+    def parse_date
+      @doc.xpath("./xmlns:date").map do |date|
+        month = date.at("./xmlns:month").text
+        year = date.at("./xmlns:year").text
+        on = "#{year}-#{Date::MONTHNAMES.index(month).to_s.rjust(2, '0')}"
+        RelatonBib::BibliographicDate.new(on: on, type: "published")
+      end
+    end
+
+    #
+    # Parse document contributors
+    #
+    # @return [Array<RelatonBib::ContributionInfo>] document contributors
+    #
+    def parse_contributor
+      @doc.xpath("./xmlns:author").map do |contributor|
+        n = contributor.at("./xmlns:name").text
+        name = RelatonBib::LocalizedString.new( n,  "en",  "Latn")
+        fname = RelatonBib::FullName.new(completename: name)
+        person = RelatonBib::Person.new(name: fname)
+        RelatonBib::ContributionInfo.new(entity: person, role: [{ type: "author" }])
+      end
+    end
+
+    #
+    # Parse document keywords
+    #
+    # @return [Array<String>] document keywords
+    #
+    def parse_keyword
+      @doc.xpath("./xmlns:keywords/xmlns:kw").map &:text
+    end
+
+    #
+    # Parse document abstract
+    #
+    # @return [Array<RelatonBib::FormattedString>] document abstract
+    #
+    def parse_abstract
+      @doc.xpath("./xmlns:abstract").map do |c|
+        RelatonBib::FormattedString.new(content: c.text, language: "en",
+                                        script: "Latn", format: "text/html")
+      end
+    end
+
+    #
+    # Parse document relations
+    #
+    # @return [Arra<RelatonBib::DocumentRelation>] document relations
+    #
+    def parse_relation
+      types = { "updates" => "updates", "obsoleted-by" => "obsoletedBy"}
+      @doc.xpath("./xmlns:updates/xmlns:doc-id|./xmlns:obsoleted-by/xmlns:doc-id").map do |r|
+        fref = RelatonBib::FormattedRef.new(content: r.text)
+        bib = IetfBibliographicItem.new(formattedref: fref)
+        RelatonBib::DocumentRelation.new(type: types[r.parent.name], bibitem: bib)
+      end
+    end
+
+    #
+    # Parse document status
+    #
+    # @return [RelatonBib::DocuemntStatus] document status
+    #
+    def parse_status
+      stage = @doc.at("./xmlns:current-status").text
+      RelatonBib::DocumentStatus.new(stage: stage)
+    end
+
+    #
+    # Parse document editorial group
+    #
+    # @return [RelatonBib::EditorialGroup] document editorial group
+    #
+    def parse_editorialgroup
+      tc = @doc.xpath("./xmlns:wg_acronym").map do |wg|
+        wg = RelatonBib::WorkGroup.new(name: wg.text)
+        RelatonBib::TechnicalCommittee.new(wg)
+      end
+      RelatonBib::EditorialGroup.new(tc)
+    end
+  end
+end

data/lib/relaton_ietf/rfc_index_entry.rb

@@ -0,0 +1,60 @@
+module RelatonIetf
+  class RfcIndexEntry
+    #
+    # Document parser initalization
+    #
+    # @param [String] doc_id document id
+    # @param [Array<String>] is_also also document ids
+    #
+    def initialize(name, doc_id, is_also)
+      @name = name
+      @shortnum = doc_id[-4..-1].sub(/^0+/, "")
+      @doc_id = doc_id
+      @is_also = is_also
+    end
+
+    #
+    # Document id
+    #
+    # @return [Strinng] document id
+    #
+    def docnumber
+      @doc_id
+    end
+
+    #
+    # Initialize document parser and run it
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    # @return [RelatonIetf:RfcIndexEntry, nil]
+    #
+    def self.parse(doc)
+      doc_id = doc.at("./xmlns:doc-id")
+      is_also = doc.xpath("./xmlns:is-also/xmlns:doc-id").map &:text
+      return unless doc_id && is_also.any?
+
+      name = doc.name.split("-").first
+      new(name, doc_id.text, is_also)
+    end
+
+    #
+    # Render document as XML
+    #
+    # @return [String] XML
+    #
+    def to_xml # rubocop:disable Metrics/MethodLength
+      Nokogiri::XML::Builder.new do |xml|
+        anchor = "#{@name.upcase}#{@shortnum}"
+        url = "https://www.rfc-editor.org/info/#{@name}#{@shortnum}"
+        xml.referencegroup("xmlns:xi" => "http://www.w3.org/2001/XInclude",
+                           anchor: anchor, target: url) do
+          @is_also.each do |did|
+            num = did[-4..-1]
+            xml["xi"].send("include", href: "https://www.rfc-editor.org/refs/bibxml/reference.RFC.#{num}.xml")
+          end
+        end
+      end.doc.root.to_xml
+    end
+  end
+end

data/lib/relaton_ietf/scrapper.rb

@@ -5,365 +5,373 @@ require "relaton_bib"
 require "relaton_ietf/ietf_bibliographic_item"
 
 module RelatonIetf
-  # rubocop:disable Metrics/ModuleLength
-
   # Scrapper module
   module Scrapper
-    GH_URL = "https://raw.githubusercontent.com/relaton/relaton-data-ietf/master/data/reference."
-    BCP_URI_PATTERN = "https://www.rfc-editor.org/info/CODE"
-
-    class << self
-      # @param text [String]
-      # @param is_relation [TrueClass, FalseClass]
-      # @return [RelatonIetf::IetfBibliographicItem]
-      def scrape_page(text, is_relation: false)
-        # Remove initial "IETF " string if specified
-        ref = text.gsub(/^IETF /, "")
-        /^(?:RFC|BCP|FYI|STD)\s(?<num>\d+)/ =~ ref
-        ref.sub! /(?<=^(?:RFC|BCP|FYI|STD)\s)(\d+)/, num.rjust(4, "0") if num
-        rfc_item ref, is_relation
-      rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
-             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
-             Net::ProtocolError, SocketError
-        raise RelatonBib::RequestError, "No document found for #{ref} reference"
-      end
-
-      # @param reference [Nokogiri::XML::Element, nil]
-      # @param is_relation [TrueClass, FalseClass]
-      # @param url [String, NilClass]
-      # @param ver [String, NilClass] Internet Draft version
-      # @return [RelatonIetf::IetfBibliographicItem]
-      def fetch_rfc(reference, is_relation: false, url: nil, ver: nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-        return unless reference
-
-        ietf_item(
-          is_relation: is_relation,
-          id: reference[:anchor],
-          type: "standard",
-          docid: docids(reference, ver),
-          status: status(reference),
-          language: [language(reference)],
-          link: link(reference, url, ver),
-          title: titles(reference),
-          formattedref: formattedref(reference),
-          abstract: abstracts(reference),
-          contributor: contributors(reference),
-          relation: relations(reference),
-          date: dates(reference),
-          series: series(reference),
-          place: ["Fremont, CA"],
-          keyword: reference.xpath("front/keyword").map(&:text),
-          doctype: doctype(reference[:anchor]),
-        )
-      end
-
-      private
-
-      # @param anchor [String]
-      # @return [String]
-      def doctype(anchor)
-        anchor&.include?("I-D") ? "internet-draft" : "rfc"
-      end
-
-      # @param reference [Nokogiri::XML::Element]
-      # @param url [String]
-      # @param ver [String, NilClass] Internet Draft version
-      # @return [Array<Hash>]
-      def link(reference, url, ver)
-        l = []
-        l << { type: "xml", content: url } if url
-        l << { type: "src", content: reference[:target] } if reference[:target]
-        if /^I-D/.match? reference[:anchor]
-          reference.xpath("format").each do |f|
-            c = ver ? f[:target].sub(/(?<=-)\d{2}(?=\.)/, ver) : f[:target]
-            l << { type: f[:type], content: c }
-          end
-        end
-        l
-      end
-
-      # @param attrs [Hash]
-      # @return [RelatonIetf::IetfBibliographicItem]
-      def ietf_item(**attrs)
-        attrs[:fetched] = Date.today.to_s unless attrs.delete(:is_relation)
-        attrs[:script] = ["Latn"]
-        RelatonIetf::IetfBibliographicItem.new **attrs
-      end
-
-      # @param ref [String]
-      # @param is_relation [Boolen, nil]
-      # @return [RelatonIetf::IetfBibliographicItem]
-      def rfc_item(ref, is_relation)
-        /(?<=-)(?<ver>\d{2})$/ =~ ref
-        if /^I-D/.match? ref
-          ref.sub! /-\d{2}/, "" if ver
-          ref.sub! /(?<=I-D\.)draft-/, ""
-        end
-
-        uri = "#{GH_URL}#{ref.sub(/\s|\u00a0/, '.')}.xml"
-        doc = Nokogiri::XML get_page(uri)
-        r = doc.at("/referencegroup", "/reference")
-        fetch_rfc r, is_relation: is_relation, url: uri, ver: ver
-      end
-
-      # @param reference [Nokogiri::XML::Element]
-      # @return [Hash]
-      def relations(reference)
-        reference.xpath("reference").map do |ref|
-          { type: "includes", bibitem: fetch_rfc(ref, is_relation: true) }
-        end
-      end
-
-      # @param uri [String]
-      # @return [String] HTTP response body
-      def get_page(uri)
-        res = Net::HTTP.get_response(URI(uri))
-        return unless res.code == "200"
-
-        #   raise RelatonBib::RequestError, "No document found at #{uri}"
-        # end
-
-        res.body
-      end
-
-      # @param reference [Nokogiri::XML::Element]
-      # @return [String]
-      def language(reference)
-        reference[:lang] || "en"
-      end
+    extend RelatonBib::BibXMLParser
+    extend Scrapper
 
-      # @param reference [Nokogiri::XML::Element]
-      # @return [Array<Hash>]
-      def titles(reference)
-        reference.xpath("./front/title").map do |title|
-          { content: title.text, language: language(reference), script: "Latn" }
-        end
-      end
+    FLAVOR = "IETF"
 
-      # @param reference [Nokogiri::XML::Element]
-      # @return [RelatonBib::FormattedRef, nil]
-      def formattedref(reference)
-        return if reference.at "./fornt/title"
-
-        cont = (reference[:anchor] || reference[:docName] || reference[:number])
-        if cont
-          RelatonBib::FormattedRef.new(
-            content: cont, language: language(reference), script: "Latn",
-          )
-        end
-      end
-
-      # @param reference [Nokogiri::XML::Element]
-      # @return [Array<RelatonBib::FormattedString>]
-      def abstracts(ref)
-        ref.xpath("./front/abstract").map do |a|
-          RelatonBib::FormattedString.new(
-            content: a.text.gsub(/\\n\\t{2,4}/, " ").strip,
-            language: language(ref), script: "Latn"
-          )
-        end
-      end
-
-      # @param reference [Nokogiri::XML::Element]
-      # @return [Array<Hash>]
-      def contributors(reference)
-        persons(reference) + organizations(reference)
-      end
-
-      # @param reference [Nokogiri::XML::Element]
-      # @return [Array<Hash{Symbol=>RelatonBib::Person,Symbol=>Array<String>}>]
-      def persons(reference)
-        reference.xpath("./front/author[@surname]|./front/author[@fullname]")
-          .map do |author|
-          entity = RelatonBib::Person.new(
-            name: full_name(author, reference),
-            affiliation: [affiliation(author)],
-            contact: contacts(author.at("./address")),
-          )
-          { entity: entity, role: [contributor_role(author)] }
-        end
-      end
-
-      # @param reference [Nokogiri::XML::Element]
-      # @return [Array<Hash{Symbol=>RelatonBib::Organization,
-      #   Symbol=>Array<String>}>]
-      def organizations(reference)
-        publisher = { entity: new_org, role: [type: "publisher"] }
-        orgs = reference.xpath("./seriesinfo").reduce([publisher]) do |mem, si|
-          next mem unless si[:stream]
-
-          mem << { entity: new_org(si[:stream], nil), role: [type: "author"] }
-        end
-        orgs + reference.xpath(
-          "front/author[not(@surname)][not(@fullname)]/organization",
-        ).map do |org|
-          { entity: new_org(org.text, nil), role: [type: "author"] }
-        end
-      end
-
-      # @param author [Nokogiri::XML::Element]
-      # @param ref [Nokogiri::XML::Element]
-      # @return [RelatonBib::FullName]
-      def full_name(author, ref)
-        lang = language ref
-        RelatonBib::FullName.new(
-          completename: localized_string(author[:fullname], lang),
-          initial: [localized_string(author[:initials], lang)].compact,
-          surname: localized_string(author[:surname], lang),
-        )
-      end
-
-      # @param content [String]
-      # @param lang [String]
-      # @return [RelatonBib::LocalizedString]
-      def localized_string(content, lang)
-        return unless content
-
-        RelatonBib::LocalizedString.new(content, lang)
-      end
-
-      # @param postal [Nokogiri::XML::Element]
-      # @return [Array<RelatonBib::Address, RelatonBib::Phone>]
-      def contacts(addr)
-        contacts = []
-        return contacts unless addr
-
-        postal = addr.at("./postal")
-        contacts << address(postal) if postal
-        add_contact(contacts, "phone", addr.at("./phone"))
-        add_contact(contacts, "email", addr.at("./email"))
-        add_contact(contacts, "uri", addr.at("./uri"))
-        contacts
-      end
-
-      # @param postal [Nokogiri::XML::Element]
-      # @rerurn [RelatonBib::Address]
-      def address(postal) # rubocop:disable Metrics/CyclomaticComplexity
-        RelatonBib::Address.new(
-          street: [(postal.at("./postalLine") || postal.at("./street"))&.text],
-          city: postal.at("./city")&.text,
-          postcode: postal.at("./code")&.text,
-          country: postal.at("./country")&.text,
-          state: postal.at("./region")&.text,
-        )
-      end
-
-      # @param type [String] allowed "phone", "email" or "uri"
-      # @param value [String]
-      def add_contact(contacts, type, value)
-        return unless value
-
-        contacts << RelatonBib::Contact.new(type: type, value: value.text)
-      end
-
-      # @param author [Nokogiri::XML::Element]
-      # @return [RelatonBib::Affiliation]
-      def affiliation(author)
-        organization = author.at("./organization")
-        org = if organization.nil? || organization&.text&.empty?
-                new_org
-              else
-                new_org organization.text, organization[:abbrev]
-              end
-        RelatonBib::Affiliation.new organization: org
-      end
+    GH_URL = "https://raw.githubusercontent.com/relaton/relaton-data-ietf/master/data/reference."
 
-      # @param name [String]
-      # @param abbr [String]
-      # @return [RelatonBib::Organization]
-      def new_org(name = "Internet Engineering Task Force", abbr = "IETF")
-        RelatonBib::Organization.new name: name, abbreviation: abbr
-      end
+    # @param text [String]
+    # @param is_relation [TrueClass, FalseClass]
+    # @return [RelatonIetf::IetfBibliographicItem]
+    def scrape_page(text, is_relation: false)
+      # Remove initial "IETF " string if specified
+      ref = text.gsub(/^IETF /, "")
+      /^(?:RFC|BCP|FYI|STD)\s(?<num>\d+)/ =~ ref
+      ref.sub!(/(?<=^(?:RFC|BCP|FYI|STD)\s)(\d+)/, num.rjust(4, "0")) if num
+      rfc_item ref, is_relation
+    rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+            Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+            Net::ProtocolError, SocketError
+      raise RelatonBib::RequestError, "No document found for #{ref} reference"
+    end
 
-      # @param author [Nokogiri::XML::Document]
-      # @return [Hash]
-      def contributor_role(author)
-        { type: author[:role] || "author" }
+    # @param reference [Nokogiri::XML::Element, nil]
+    # @param is_relation [Boolean] don't add fetched date for relation
+    # @param url [String, NilClass]
+    # @param ver [String, NilClass] Internet Draft version
+    # @return [RelatonBib::tfBibliographicItem]
+    # def fetch_rfc(reference, is_relation: false, url: nil, ver: nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    #   return unless reference
+
+    #   hash = {
+    #     id: reference[:anchor],
+    #     type: "standard",
+    #     docid: docids(reference, ver),
+    #     status: status(reference),
+    #     language: [language(reference)],
+    #     script: ["Latn"],
+    #     link: link(reference, url, ver),
+    #     title: titles(reference),
+    #     formattedref: formattedref(reference),
+    #     abstract: abstracts(reference),
+    #     contributor: contributors(reference),
+    #     relation: relations(reference),
+    #     date: dates(reference),
+    #     series: series(reference),
+    #     keyword: reference.xpath("front/keyword").map(&:text),
+    #     doctype: doctype(reference[:anchor]),
+    #   }
+    #   hash[:fetched] = Date.today.to_s unless is_relation
+    #   bib_item(**hash)
+    # end
+
+    private
+
+    # @param anchor [String]
+    # @return [String]
+    # def doctype(anchor)
+    #   anchor&.include?("I-D") ? "internet-draft" : "rfc"
+    # end
+
+    # @param reference [Nokogiri::XML::Element]
+    # @param url [String]
+    # @param ver [String, NilClass] Internet Draft version
+    # @return [Array<Hash>]
+    # def link(reference, url, ver)
+    #   l = []
+    #   l << { type: "xml", content: url } if url
+    #   l << { type: "src", content: reference[:target] } if reference[:target]
+    #   if /^I-D/.match? reference[:anchor]
+    #     reference.xpath("format").each do |f|
+    #       c = ver ? f[:target].sub(/(?<=-)\d{2}(?=\.)/, ver) : f[:target]
+    #       l << { type: f[:type], content: c }
+    #     end
+    #   end
+    #   l
+    # end
+
+    # @param attrs [Hash]
+    # @return [RelatonIetf::IetfBibliographicItem]
+    def bib_item(**attrs)
+      unless attrs.delete(:is_relation)
+        attrs[:fetched] = Date.today.to_s
+        attrs[:place] = ["Fremont, CA"]
       end
+      RelatonIetf::IetfBibliographicItem.new(**attrs)
+    end
 
-      def month(mon)
-        return 1 if !mon || mon.empty?
-        return mon if /^\d+$/.match? mon
-
-        Date::MONTHNAMES.index(mon)
+    # @param ref [String]
+    # @param is_relation [Boolen, nil]
+    # @return [RelatonIetf::IetfBibliographicItem]
+    def rfc_item(ref, is_relation)
+      /(?<=-)(?<ver>\d{2})$/ =~ ref
+      if /^I-D/.match? ref
+        ref.sub!(/-\d{2}/, "") if ver
+        ref.sub!(/(?<=I-D\.)draft-/, "")
       end
 
-      #
-      # Extract date from reference.
-      #
-      # @param reference [Nokogiri::XML::Element]
-      # @return [Array<RelatonBib::BibliographicDate>] published data.
-      #
-      def dates(reference)
-        return unless (date = reference.at "./front/date")
-
-        d = [date[:year], month(date[:month]),
-             (date[:day] || 1)].compact.join "-"
-        date = Time.parse(d).strftime "%Y-%m-%d"
-        [RelatonBib::BibliographicDate.new(type: "published", on: date)]
-      end
+      uri = "#{GH_URL}#{ref.sub(/\s|\u00a0/, '.')}.xml"
+      # doc = Nokogiri::XML get_page(uri)
+      # r = doc.at("/referencegroup", "/reference")
+      # fetch_rfc r, is_relation: is_relation, url: uri, ver: ver
+      parse get_page(uri), url: uri, is_relation: is_relation, ver: ver
+    end
 
-      #
-      # Extract document identifiers from reference
-      #
-      # @param reference [Nokogiri::XML::Element]
-      # @param ver [String, NilClass] Internet Draft version
-      #
-      # @return [Array<RelatonBib::DocumentIdentifier>]
-      #
-      def docids(reference, ver) # rubocop:disable Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize
-        id = (reference[:anchor] || reference[:docName] || reference[:number])
-        ret = []
-        if id
-          ret << RelatonBib::DocumentIdentifier.new(
-            type: "IETF", id: id.sub(/^(RFC)/, "\\1 "),
-          )
-        end
-        if (id = reference[:anchor])
-          ret << RelatonBib::DocumentIdentifier.new(type: "rfc-anchor", id: id)
-        end
-        ret + reference.xpath("./seriesInfo").map do |si|
-          next unless ["DOI", "Internet-Draft"].include? si[:name]
-
-          id = si[:value]
-          id.sub! /(?<=-)\d{2}$/, ver if ver && si[:name] == "Internet-Draft"
-          RelatonBib::DocumentIdentifier.new(id: id, type: si[:name])
-        end.compact
-      end
+    # @param reference [Nokogiri::XML::Element]
+    # @return [Hash]
+    # def relations(reference)
+    #   reference.xpath("reference").map do |ref|
+    #     { type: "includes", bibitem: fetch_rfc(ref, is_relation: true) }
+    #   end
+    # end
+
+    # @param uri [String]
+    # @return [String] HTTP response body
+    def get_page(uri)
+      res = Net::HTTP.get_response(URI(uri))
+      return unless res.code == "200"
+
+      res.body
+    end
 
-      #
-      # Extract series form reference
-      # @param reference [Nokogiri::XML::Element]
-      #
-      # @return [Array<RelatonBib::Series>]
-      #
-      def series(reference)
-        reference.xpath("./seriesInfo").map do |si|
-          next if si[:name] == "DOI" || si[:stream] || si[:status]
-
-          RelatonBib::Series.new(
-            title: RelatonBib::TypedTitleString.new(
-              content: si[:name], language: language(reference), script: "Latn",
-            ),
-            number: si[:value],
-            type: "main",
-          )
-        end.compact
-      end
+    # @param [RelatonBib::WorkGroup]
+    # @return [RelatonIetf::Committee]
+    def committee(wgr)
+      Committee.new wgr
+    end
 
-      #
-      # extract status
-      # @param reference [Nokogiri::XML::Element]
-      #
-      # @return [RelatonBib::DocumentStatus]
-      #
-      def status(reference)
-        st = reference.at("./seriesinfo[@status]")
-        return unless st
-
-        RelatonBib::DocumentStatus.new(stage: st[:status])
-      end
+    # @param reference [Nokogiri::XML::Element]
+    # @return [String]
+    # def language(reference)
+    #   reference[:lang] || "en"
+    # end
+
+    # @param reference [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    # def titles(reference)
+    #   reference.xpath("./front/title").map do |title|
+    #     { content: title.text, language: language(reference), script: "Latn" }
+    #   end
+    # end
+
+    # @param reference [Nokogiri::XML::Element]
+    # @return [RelatonBib::FormattedRef, nil]
+    # def formattedref(reference)
+    #   return if reference.at "./front/title"
+
+    #   cont = (reference[:anchor] || reference[:docName] || reference[:number])
+    #   if cont
+    #     RelatonBib::FormattedRef.new(
+    #       content: cont, language: language(reference), script: "Latn",
+    #     )
+    #   end
+    # end
+
+    # @param reference [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::FormattedString>]
+    # def abstracts(ref)
+    #   ref.xpath("./front/abstract").map do |a|
+    #     RelatonBib::FormattedString.new(
+    #       content: a.text.gsub(/\\n\\t{2,4}/, " ").strip,
+    #       language: language(ref), script: "Latn"
+    #     )
+    #   end
+    # end
+
+    # @param reference [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def contributors(reference)
+      [{ entity: new_org("Internet Engineering Task Force", "IETF"), role: [type: "publisher"] }] + super
+      # persons(reference) + organizations(reference)
     end
+
+    # @param reference [Nokogiri::XML::Element]
+    # @return [Array<Hash{Symbol=>RelatonBib::Person,Symbol=>Array<String>}>]
+    # def persons(reference)
+    #   reference.xpath("./front/author[@surname]|./front/author[@fullname]")
+    #     .map do |author|
+    #     entity = RelatonBib::Person.new(
+    #       name: full_name(author, reference),
+    #       affiliation: affiliation(author),
+    #       contact: contacts(author.at("./address")),
+    #     )
+    #     { entity: entity, role: [contributor_role(author)] }
+    #   end
+    # end
+
+    # @param reference [Nokogiri::XML::Element]
+    # @return [Array<Hash{Symbol=>RelatonBib::Organization,
+    #   Symbol=>Array<String>}>]
+    # def organizations(reference)
+    #   publisher = { entity: new_org, role: [type: "publisher"] }
+    #   orgs = reference.xpath("./seriesinfo").reduce([publisher]) do |mem, si|
+    #     next mem unless si[:stream]
+
+    #     mem << { entity: new_org(si[:stream], nil), role: [type: "author"] }
+    #   end
+    #   orgs + reference.xpath(
+    #     "front/author[not(@surname)][not(@fullname)]/organization",
+    #   ).map do |org|
+    #     { entity: new_org(org.text, nil), role: [type: "author"] }
+    #   end
+    # end
+
+    # @param author [Nokogiri::XML::Element]
+    # @param ref [Nokogiri::XML::Element]
+    # @return [RelatonBib::FullName]
+    # def full_name(author, ref)
+    #   lang = language ref
+    #   RelatonBib::FullName.new(
+    #     completename: localized_string(author[:fullname], lang),
+    #     initial: [localized_string(author[:initials], lang)].compact,
+    #     surname: localized_string(author[:surname], lang),
+    #   )
+    # end
+
+    # @param content [String]
+    # @param lang [String]
+    # @return [RelatonBib::LocalizedString]
+    # def localized_string(content, lang)
+    #   return unless content
+
+    #   RelatonBib::LocalizedString.new(content, lang)
+    # end
+
+    # @param postal [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::Address, RelatonBib::Phone>]
+    # def contacts(addr)
+    #   contacts = []
+    #   return contacts unless addr
+
+    #   postal = addr.at("./postal")
+    #   contacts << address(postal) if postal
+    #   add_contact(contacts, "phone", addr.at("./phone"))
+    #   add_contact(contacts, "email", addr.at("./email"))
+    #   add_contact(contacts, "uri", addr.at("./uri"))
+    #   contacts
+    # end
+
+    # @param postal [Nokogiri::XML::Element]
+    # @rerurn [RelatonBib::Address]
+    # def address(postal) # rubocop:disable Metrics/CyclomaticComplexity
+    #   RelatonBib::Address.new(
+    #     street: [(postal.at("./postalLine") || postal.at("./street"))&.text],
+    #     city: postal.at("./city")&.text,
+    #     postcode: postal.at("./code")&.text,
+    #     country: postal.at("./country")&.text,
+    #     state: postal.at("./region")&.text,
+    #   )
+    # end
+
+    # @param type [String] allowed "phone", "email" or "uri"
+    # @param value [String]
+    # def add_contact(contacts, type, value)
+    #   return unless value
+
+    #   contacts << RelatonBib::Contact.new(type: type, value: value.text)
+    # end
+
+    # @param author [Nokogiri::XML::Element]
+    # @return [RelatonBib::Affiliation]
+    # def affiliation(author)
+    #   organization = author.at("./organization")
+    #   org = if organization.nil? || organization&.text&.empty?
+    #           new_org
+    #         else
+    #           new_org organization.text, organization[:abbrev]
+    #         end
+    #   RelatonBib::Affiliation.new organization: org
+    # end
+
+    # @param name [String]
+    # @param abbr [String]
+    # @return [RelatonBib::Organization]
+    # def new_org(name = "Internet Engineering Task Force", abbr = "IETF")
+    #   RelatonBib::Organization.new name: name, abbreviation: abbr
+    # end
+
+    # @param author [Nokogiri::XML::Document]
+    # @return [Hash]
+    # def contributor_role(author)
+    #   { type: author[:role] || "author" }
+    # end
+
+    # def month(mon)
+    #   return 1 if !mon || mon.empty?
+    #   return mon if /^\d+$/.match? mon
+
+    #   Date::MONTHNAMES.index(mon)
+    # end
+
+    #
+    # Extract date from reference.
+    #
+    # @param reference [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::BibliographicDate>] published data.
+    #
+    # def dates(reference)
+    #   return unless (date = reference.at "./front/date")
+
+    #   d = [date[:year], month(date[:month]),
+    #        (date[:day] || 1)].compact.join "-"
+    #   date = Time.parse(d).strftime "%Y-%m-%d"
+    #   [RelatonBib::BibliographicDate.new(type: "published", on: date)]
+    # end
+
+    #
+    # Extract document identifiers from reference
+    #
+    # @param reference [Nokogiri::XML::Element]
+    # @param ver [String, NilClass] Internet Draft version
+    #
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    #
+    # def docids(reference, ver) # rubocop:disable Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/AbcSize
+    #   id = (reference[:anchor] || reference[:docName] || reference[:number])
+    #   ret = []
+    #   if id
+    #     ret << RelatonBib::DocumentIdentifier.new(
+    #       type: "IETF", id: id.sub(/^(RFC)/, "\\1 "),
+    #     )
+    #   end
+    #   if (id = reference[:anchor])
+    #     ret << RelatonBib::DocumentIdentifier.new(type: "rfc-anchor", id: id)
+    #   end
+    #   names = ["DOI", "Internet-Draft"]
+    #   ret + reference.xpath("./seriesInfo").map do |si|
+    #     next unless names.include? si[:name]
+
+    #     id = si[:value]
+    #     id.sub!(/(?<=-)\d{2}$/, ver) if ver && si[:name] == "Internet-Draft"
+    #     RelatonBib::DocumentIdentifier.new(id: id, type: si[:name])
+    #   end.compact
+    # end
+
+    #
+    # Extract series form reference
+    # @param reference [Nokogiri::XML::Element]
+    #
+    # @return [Array<RelatonBib::Series>]
+    #
+    # def series(reference)
+    #   reference.xpath("./seriesInfo").map do |si|
+    #     next if si[:name] == "DOI" || si[:stream] || si[:status]
+
+    #     RelatonBib::Series.new(
+    #       title: RelatonBib::TypedTitleString.new(
+    #         content: si[:name], language: language(reference), script: "Latn",
+    #       ),
+    #       number: si[:value],
+    #       type: "main",
+    #     )
+    #   end.compact
+    # end
+
+    #
+    # extract status
+    # @param reference [Nokogiri::XML::Element]
+    #
+    # @return [RelatonBib::DocumentStatus]
+    #
+    # def status(reference)
+    #   st = reference.at("./seriesinfo[@status]")
+    #   return unless st
+
+    #   RelatonBib::DocumentStatus.new(stage: st[:status])
+    # end
   end
-  # rubocop:enable Metrics/ModuleLength
 end

data/lib/relaton_ietf/version.rb

@@ -1,3 +1,3 @@
 module RelatonIetf
-  VERSION = "1.9.0".freeze
+  VERSION = "1.9.4".freeze
 end

data/lib/relaton_ietf/xml_parser.rb

@@ -9,7 +9,7 @@ module RelatonIetf
       # @param item_hash [Hash]
       # @return [RelatonIetf::IetfBibliographicItem]
       def bib_item(item_hash)
-        IetfBibliographicItem.new **item_hash
+        IetfBibliographicItem.new(**item_hash)
       end
     end
   end

data/lib/relaton_ietf.rb

@@ -5,6 +5,8 @@ require "relaton_ietf/ietf_bibliography"
 require "relaton_ietf/ietf_bibliographic_item"
 require "relaton_ietf/xml_parser"
 require "relaton_ietf/hash_converter"
+require "relaton_ietf/committee"
+require "relaton_ietf/data_fetcher"
 
 require "relaton/provider_ietf"
 

data/relaton_ietf.gemspec

@@ -38,5 +38,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 
-  spec.add_dependency "relaton-bib", "~> 1.9.0"
+  spec.add_dependency "relaton-bib", ">= 1.9.8"
+  spec.add_dependency "zlib", "~> 1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-ietf
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.9.4
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-26 00:00:00.000000000 Z
+date: 2021-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -124,18 +124,32 @@ dependencies:
         version: '0'
 - !ruby/object:Gem::Dependency
   name: relaton-bib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.8
+- !ruby/object:Gem::Dependency
+  name: zlib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.1.0
 description: "RelatonIetf: retrieve IETF Standards for bibliographic use \nusing the
   BibliographicItem model.\n\nFormerly known as rfcbib.\n"
 email:
@@ -163,10 +177,14 @@ files:
 - grammars/reqt.rng
 - lib/relaton/provider_ietf.rb
 - lib/relaton_ietf.rb
+- lib/relaton_ietf/committee.rb
+- lib/relaton_ietf/data_fetcher.rb
 - lib/relaton_ietf/hash_converter.rb
 - lib/relaton_ietf/ietf_bibliographic_item.rb
 - lib/relaton_ietf/ietf_bibliography.rb
 - lib/relaton_ietf/processor.rb
+- lib/relaton_ietf/rfc_entry.rb
+- lib/relaton_ietf/rfc_index_entry.rb
 - lib/relaton_ietf/scrapper.rb
 - lib/relaton_ietf/version.rb
 - lib/relaton_ietf/xml_parser.rb