relaton-ietf 1.9.1 → 1.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ed659b3ac3a91f53fa5872cb839be97adeba30e6eba56f211f140e55b5836d8
-  data.tar.gz: 8b1c58e40158c126f11ab25c92ca99230a3e00064b8d0fb20ad250c51c01e7b8
+  metadata.gz: 662721a37445e18e6d5c2cce9f99cc3a62fe3d59a20870d00bf2f840bb645fd7
+  data.tar.gz: 4129066b66f1fc0391ac2d06db9735dfe86c288be0004276be0092e06e2cf461
 SHA512:
-  metadata.gz: 163f11e5a49d8dd44d6592c1741281b8756f505ac49feb59d3a508ec2ad7340498db0a5d71a1ccfbbc8c4de385ee967388119238f0c4f94641a839623fe19c65
-  data.tar.gz: 0a86ac885aca2d200e5d64cdee74e0b4b5e6d42f77a5e9fb60c233e77e19c176137ebc8220a7397e56e6564edcd58c09933553810a4265597fbc7e64c02a173b
+  metadata.gz: 797bacba502406e48b573ab58ca9ec413dd8e51b5fef5c28b4676e7ba233cb2e4db3ffffefcff233a2f8a440bee9296083c11420684f9307a4b2bbf7e85c68b5
+  data.tar.gz: e0fcc1ac666e54a869aec07fa5fe6fd5397b04e6afced205cbee57366310b7fccf23840b1d724a30e7c8f2781fb8d97f4a5ad36f9637510b5466f0ba14c1621b

data/.rubocop.yml

@@ -2,6 +2,8 @@
 # https://github.com/riboseinc/oss-guides
 # All project-specific additions and overrides should be specified in this file.
 
+require: rubocop-rails
+
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:

data/README.adoc

@@ -133,6 +133,41 @@ RelatonIetf::IetfBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+There is a IETF datasets what can be converted into RelatonXML/BibXML/BibYAML formats:
+
+- `ietf-rfcsubseries` - https://www.rfc-editor.org/rfc-index.xml (`<bcp-entry>`, `<fyi-entry>`, `<std-entry>`)
+- `ietf-internet-drafts` - https://www.ietf.org/lib/dt/sprint/bibxml-ids.tgz
+- `ietf-rfc-entries` - https://www.rfc-editor.org/rfc-index.xml (`<rfc-entry>`)
+
+The method `RelatonIetf::DataFetcher.fetch(source, output: "data", format: "yaml")` converts all the documents from the dataset and save them to the `./data` folder in YAML format.
+
+Arguments:
+
+- `source` - dataset name (`ietf-rfcsubseries` or `ietf-internet-drafts`)
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml`, `bibxml` (default `yaml`).
+
+For `ietf-rfcsubseries` dataset only special XML format is supported:
+
+[sourse.xml]
+----
+<referencegroup anchor="BCP14" target="https://www.rfc-editor.org/info/bcp14">
+  <xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.2119.xml" />
+  <xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.8174.xml" />
+</referencegroup>
+----
+
+[source,ruby]
+----
+RelatonIetf::DataFetcher.fetch "ietf-internet-drafts"
+Started at: 2021-12-17 10:23:20 +0100
+Stopped at: 2021-12-17 10:29:19 +0100
+Done in: 360 sec.
+=> nil
+----
+
 == Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/metanorma/relaton-ietf.

data/lib/relaton_ietf/data_fetcher.rb

@@ -0,0 +1,133 @@
+require "rubygems"
+require "rubygems/package"
+require "zlib"
+require "relaton_ietf/rfc_index_entry"
+require "relaton_ietf/rfc_entry"
+
+module RelatonIetf
+  class DataFetcher
+    #
+    # Data fetcher initializer
+    #
+    # @param [String] source source name
+    # @param [String] output directory to save files
+    # @param [String] format format of output files (xml, yaml, bibxml);
+    #   for ietf-rfcsubseries source only: xml
+    #
+    def initialize(source, output, format)
+      @source = source
+      @output = output
+      @format = source == "ietf-rfcsubseries" ? "rfcxml" : format
+      @ext = @format.sub(/^bib|^rfc/, "")
+      @files = []
+    end
+
+    #
+    # Initialize fetcher and run fetch
+    #
+    # @param [String] source source name
+    # @param [Strin] output directory to save files, default: "data"
+    # @param [Strin] format format of output files (xml, yaml, bibxml);
+    #   default: yaml; for ietf-rfcsubseries source only: xml
+    #
+    def self.fetch(source, output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(source, output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Fetch documents
+    #
+    def fetch
+      case @source
+      when "ietf-rfcsubseries" then fetch_ieft_rfcsubseries
+      when "ietf-internet-drafts" then fetch_ieft_internet_drafts
+      when "ietf-rfc-entries" then fetch_ieft_rfcs
+      end
+    end
+
+    #
+    # Fetches ietf-rfcsubseries documents
+    #
+    def fetch_ieft_rfcsubseries
+      rfc_index.xpath("xmlns:bcp-entry|xmlns:fyi-entry|xmlns:std-entry").each do |doc|
+        save_doc RfcIndexEntry.parse(doc)
+      end
+    end
+
+    #
+    # Fetches ietf-internet-drafts documents
+    #
+    def fetch_ieft_internet_drafts # rubocop:disable Metrics/MethodLength
+      gz = OpenURI.open_uri("https://www.ietf.org/lib/dt/sprint/bibxml-ids.tgz")
+      z = Zlib::GzipReader.new(gz)
+      io = StringIO.new(z.read)
+      z.close
+      Gem::Package::TarReader.new io do |tar|
+        tar.each do |tarfile|
+          next if tarfile.directory?
+
+          save_doc RelatonBib::BibXMLParser.parse(tarfile.read)
+        end
+      end
+    end
+
+    def fetch_ieft_rfcs
+      rfc_index.xpath("xmlns:rfc-entry").each do |doc|
+        save_doc RfcEntry.parse(doc)
+      end
+    end
+
+    def rfc_index
+      uri = URI "https://www.rfc-editor.org/rfc-index.xml"
+      Nokogiri::XML(Net::HTTP.get(uri)).at("/xmlns:rfc-index")
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonIetf::RfcIndexEntry, nil] rfc index entry
+    #
+    def save_doc(entry) # rubocop:disable Metrics/MethodLength
+      return unless entry
+
+      c = case @format
+          when "xml" then entry.to_xml(bibdata: true)
+          when "yaml" then entry.to_hash.to_yaml
+          when "rfcxml" then entry.to_xml
+          else entry.send("to_#{@format}")
+          end
+      file = file_name entry
+      if @files.include? file
+        warn "File #{file} already exists. Document: #{entry.docnumber}"
+      else
+        @files << file
+      end
+      File.write file, c, encoding: "UTF-8"
+    end
+
+    #
+    # Generate file name
+    #
+    # @param [RelatonIetf::RfcIndexEntry] entry
+    #
+    # @return [String] file name
+    #
+    def file_name(entry)
+      id = if entry.respond_to? :docidentifier
+             entry.docidentifier.detect { |i| i.type == "Internet-Draft" }&.id
+           end
+      id ||= entry.docnumber
+      if @source == "ietf-internet-drafts" then id.downcase!
+      else id.upcase!
+      end
+      name = id.gsub(/[\s,:\/]/, "_").squeeze("_")
+      File.join @output, "#{name}.#{@ext}"
+    end
+  end
+end

data/lib/relaton_ietf/processor.rb

@@ -3,11 +3,12 @@ require "relaton_ietf/xml_parser"
 
 module RelatonIetf
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_ietf
       @prefix = "IETF"
       @defaultprefix = /^RFC /
       @idtype = "IETF"
+      @datasets = %w[ietf-rfcsubseries ietf-internet-drafts ietf-rfc-entries]
     end
 
     # @param code [String]
@@ -18,6 +19,18 @@ module RelatonIetf
       ::RelatonIetf::IetfBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from https://www.rfc-editor.org/rfc-index.xml
+    #
+    # @param [String] source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(source, opts)
+      DataFetcher.fetch(source, **opts)
+    end
+
     # @param xml [String]
     # @return [RelatonIetf::IetfBibliographicItem]
     def from_xml(xml)

data/lib/relaton_ietf/rfc_entry.rb

@@ -0,0 +1,186 @@
+module RelatonIetf
+  class RfcEntry
+    #
+    # Initalize parser
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    def initialize(doc)
+      @doc = doc
+    end
+
+    #
+    # Initialize parser & parse document
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    # @return [RelatonIetf::IetfBibliographicItem] bib item
+    #
+    def self.parse(doc)
+      new(doc).parse
+    end
+
+    #
+    # Parse document
+    #
+    # @return [RelatonIetf::IetfBibliographicItem] bib item
+    #
+    def parse # rubocop:disable Metrics/MethodLength
+      IetfBibliographicItem.new(
+        type: "standard",
+        language: ["en"],
+        script: ["Latn"],
+        fetched: Date.today.to_s,
+        docid: parse_docid,
+        docnumber: code,
+        title: parse_title,
+        link: parse_link,
+        date: parse_date,
+        contributor: parse_contributor,
+        keyword: parse_keyword,
+        abstract: parse_abstract,
+        relation: parse_relation,
+        status: parse_status,
+        editorialgroup: parse_editorialgroup,
+      )
+    end
+
+    #
+    # Parse document identifiers
+    #
+    # @return [Array<RelatonBib::DocumentIdettifier>] document identifiers
+    #
+    def parse_docid
+      ids = [RelatonBib::DocumentIdentifier.new(id: pub_id, type: "IETF")]
+      doi = @doc.at("./xmlns:doi").text
+      ids << RelatonBib::DocumentIdentifier.new(id: doi, type: "DOI")
+      ids
+    end
+
+    #
+    # Parse document title
+    #
+    # @return [Array<RelatonBib::TypedTileString>] document title
+    #
+    def parse_title
+      content = @doc.at("./xmlns:title").text
+      [RelatonBib::TypedTitleString.new(content: content, type: "main")]
+    end
+
+    #
+    # Create PubID
+    #
+    # @return [String] PubID
+    #
+    def pub_id
+      "IETF #{code.sub(/^(RFC)(\d+)/, '\1 \2')}"
+    end
+
+    #
+    # Parse document code
+    #
+    # @return [String] document code
+    #
+    def code
+      @doc.at("./xmlns:doc-id").text
+    end
+
+    #
+    # Create link
+    #
+    # @return [Array<RelatonBib::TypedUri>] 
+    #
+    def parse_link
+      num = code[-4..-1].sub(/^0+/, "")
+      url = "https://www.rfc-editor.org/info/rfc#{num}"
+      [RelatonBib::TypedUri.new(content: url, type: "src")]
+    end
+
+    #
+    # Parse document date
+    #
+    # @return [Array<RelatonBib::BibliographicDate>] document date
+    #
+    def parse_date
+      @doc.xpath("./xmlns:date").map do |date|
+        month = date.at("./xmlns:month").text
+        year = date.at("./xmlns:year").text
+        on = "#{year}-#{Date::MONTHNAMES.index(month).to_s.rjust(2, '0')}"
+        RelatonBib::BibliographicDate.new(on: on, type: "published")
+      end
+    end
+
+    #
+    # Parse document contributors
+    #
+    # @return [Array<RelatonBib::ContributionInfo>] document contributors
+    #
+    def parse_contributor
+      @doc.xpath("./xmlns:author").map do |contributor|
+        n = contributor.at("./xmlns:name").text
+        name = RelatonBib::LocalizedString.new( n,  "en",  "Latn")
+        fname = RelatonBib::FullName.new(completename: name)
+        person = RelatonBib::Person.new(name: fname)
+        RelatonBib::ContributionInfo.new(entity: person, role: [{ type: "author" }])
+      end
+    end
+
+    #
+    # Parse document keywords
+    #
+    # @return [Array<String>] document keywords
+    #
+    def parse_keyword
+      @doc.xpath("./xmlns:keywords/xmlns:kw").map &:text
+    end
+
+    #
+    # Parse document abstract
+    #
+    # @return [Array<RelatonBib::FormattedString>] document abstract
+    #
+    def parse_abstract
+      @doc.xpath("./xmlns:abstract").map do |c|
+        RelatonBib::FormattedString.new(content: c.text, language: "en",
+                                        script: "Latn", format: "text/html")
+      end
+    end
+
+    #
+    # Parse document relations
+    #
+    # @return [Arra<RelatonBib::DocumentRelation>] document relations
+    #
+    def parse_relation
+      types = { "updates" => "updates", "obsoleted-by" => "obsoletedBy"}
+      @doc.xpath("./xmlns:updates/xmlns:doc-id|./xmlns:obsoleted-by/xmlns:doc-id").map do |r|
+        fref = RelatonBib::FormattedRef.new(content: r.text)
+        bib = IetfBibliographicItem.new(formattedref: fref)
+        RelatonBib::DocumentRelation.new(type: types[r.parent.name], bibitem: bib)
+      end
+    end
+
+    #
+    # Parse document status
+    #
+    # @return [RelatonBib::DocuemntStatus] document status
+    #
+    def parse_status
+      stage = @doc.at("./xmlns:current-status").text
+      RelatonBib::DocumentStatus.new(stage: stage)
+    end
+
+    #
+    # Parse document editorial group
+    #
+    # @return [RelatonBib::EditorialGroup] document editorial group
+    #
+    def parse_editorialgroup
+      tc = @doc.xpath("./xmlns:wg_acronym").map do |wg|
+        wg = RelatonBib::WorkGroup.new(name: wg.text)
+        RelatonBib::TechnicalCommittee.new(wg)
+      end
+      RelatonBib::EditorialGroup.new(tc)
+    end
+  end
+end

data/lib/relaton_ietf/rfc_index_entry.rb

@@ -0,0 +1,60 @@
+module RelatonIetf
+  class RfcIndexEntry
+    #
+    # Document parser initalization
+    #
+    # @param [String] doc_id document id
+    # @param [Array<String>] is_also also document ids
+    #
+    def initialize(name, doc_id, is_also)
+      @name = name
+      @shortnum = doc_id[-4..-1].sub(/^0+/, "")
+      @doc_id = doc_id
+      @is_also = is_also
+    end
+
+    #
+    # Document id
+    #
+    # @return [Strinng] document id
+    #
+    def docnumber
+      @doc_id
+    end
+
+    #
+    # Initialize document parser and run it
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    # @return [RelatonIetf:RfcIndexEntry, nil]
+    #
+    def self.parse(doc)
+      doc_id = doc.at("./xmlns:doc-id")
+      is_also = doc.xpath("./xmlns:is-also/xmlns:doc-id").map &:text
+      return unless doc_id && is_also.any?
+
+      name = doc.name.split("-").first
+      new(name, doc_id.text, is_also)
+    end
+
+    #
+    # Render document as XML
+    #
+    # @return [String] XML
+    #
+    def to_xml # rubocop:disable Metrics/MethodLength
+      Nokogiri::XML::Builder.new do |xml|
+        anchor = "#{@name.upcase}#{@shortnum}"
+        url = "https://www.rfc-editor.org/info/#{@name}#{@shortnum}"
+        xml.referencegroup("xmlns:xi" => "http://www.w3.org/2001/XInclude",
+                           anchor: anchor, target: url) do
+          @is_also.each do |did|
+            num = did[-4..-1]
+            xml["xi"].send("include", href: "https://www.rfc-editor.org/refs/bibxml/reference.RFC.#{num}.xml")
+          end
+        end
+      end.doc.root.to_xml
+    end
+  end
+end

data/lib/relaton_ietf/scrapper.rb

@@ -174,9 +174,10 @@ module RelatonIetf
 
     # @param reference [Nokogiri::XML::Element]
     # @return [Array<Hash>]
-    # def contributors(reference)
-    #   persons(reference) + organizations(reference)
-    # end
+    def contributors(reference)
+      [{ entity: new_org("Internet Engineering Task Force", "IETF"), role: [type: "publisher"] }] + super
+      # persons(reference) + organizations(reference)
+    end
 
     # @param reference [Nokogiri::XML::Element]
     # @return [Array<Hash{Symbol=>RelatonBib::Person,Symbol=>Array<String>}>]
@@ -185,7 +186,7 @@ module RelatonIetf
     #     .map do |author|
     #     entity = RelatonBib::Person.new(
     #       name: full_name(author, reference),
-    #       affiliation: [affiliation(author)],
+    #       affiliation: affiliation(author),
     #       contact: contacts(author.at("./address")),
     #     )
     #     { entity: entity, role: [contributor_role(author)] }

data/lib/relaton_ietf/version.rb

@@ -1,3 +1,3 @@
 module RelatonIetf
-  VERSION = "1.9.1".freeze
+  VERSION = "1.9.5".freeze
 end

data/lib/relaton_ietf.rb

@@ -6,6 +6,7 @@ require "relaton_ietf/ietf_bibliographic_item"
 require "relaton_ietf/xml_parser"
 require "relaton_ietf/hash_converter"
 require "relaton_ietf/committee"
+require "relaton_ietf/data_fetcher"
 
 require "relaton/provider_ietf"
 

data/relaton_ietf.gemspec

@@ -38,5 +38,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 
-  spec.add_dependency "relaton-bib", ">= 1.9.5"
+  spec.add_dependency "relaton-bib", ">= 1.9.8"
+  spec.add_dependency "zlib", "~> 1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-ietf
 version: !ruby/object:Gem::Version
-  version: 1.9.1
+  version: 1.9.5
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-14 00:00:00.000000000 Z
+date: 2021-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -128,14 +128,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.5
+        version: 1.9.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.5
+        version: 1.9.8
+- !ruby/object:Gem::Dependency
+  name: zlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
 description: "RelatonIetf: retrieve IETF Standards for bibliographic use \nusing the
   BibliographicItem model.\n\nFormerly known as rfcbib.\n"
 email:
@@ -164,10 +178,13 @@ files:
 - lib/relaton/provider_ietf.rb
 - lib/relaton_ietf.rb
 - lib/relaton_ietf/committee.rb
+- lib/relaton_ietf/data_fetcher.rb
 - lib/relaton_ietf/hash_converter.rb
 - lib/relaton_ietf/ietf_bibliographic_item.rb
 - lib/relaton_ietf/ietf_bibliography.rb
 - lib/relaton_ietf/processor.rb
+- lib/relaton_ietf/rfc_entry.rb
+- lib/relaton_ietf/rfc_index_entry.rb
 - lib/relaton_ietf/scrapper.rb
 - lib/relaton_ietf/version.rb
 - lib/relaton_ietf/xml_parser.rb