relaton-ietf 1.9.2 → 1.9.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e16b219a2bc4343e1f47d35629131477d8dd88199fc012b252d2404277b6a2b2
-  data.tar.gz: ec0c784f65452bd35f9d9a43176654e3a1da583fdd028e001cb594707a0252c7
+  metadata.gz: 05d7da0539d628af6e8dbe016497f640fe1a238c164e280e6d9bae3152de3bb5
+  data.tar.gz: 4d7da3f63909c912630f6b2d2ac7c2224219662a6cd8ce840f4156bb59072e18
 SHA512:
-  metadata.gz: b3f427e275c57c2de8d6c6fb4d27fb330c0674facb4b02e2eab172228aa758f27343485dba4306efa68b5fd72c799cce6e3de33d79d5183de524f8cd4ec4ea69
-  data.tar.gz: 926b1b544cbad65a57561f0566f644795c0d31b2173aadcbe054d4e305e8028796fe01a31a1ca3deae5aac0e1db7f6f1fa057c8113ad6f827c2e8d1b7a04db53
+  metadata.gz: 688aefaa75847bfbb3dd9dc41dd91367b0e5749d78f2ef0f93296270d25165b69708a5a20367231df67525d536755f5eeef9c9f1f242a3e7b895381fac41f427
+  data.tar.gz: adc30f681ff6be584b9025aaad6d14ffa728a1bb82b10caac5d88a14954017dfc9a6328fd34c9f13176c3e9a2a3496b676d18d8405070bf00b92c5b15536f380

data/.rubocop.yml

@@ -2,6 +2,8 @@
 # https://github.com/riboseinc/oss-guides
 # All project-specific additions and overrides should be specified in this file.
 
+require: rubocop-rails
+
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:

data/README.adoc

@@ -133,6 +133,41 @@ RelatonIetf::IetfBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+There is a IETF datasets what can be converted into RelatonXML/BibXML/BibYAML formats:
+
+- `ietf-rfcsubseries` - https://www.rfc-editor.org/rfc-index.xml (`<bcp-entry>`, `<fyi-entry>`, `<std-entry>`)
+- `ietf-internet-drafts` - https://www.ietf.org/lib/dt/sprint/bibxml-ids.tgz
+- `ietf-rfc-entries` - https://www.rfc-editor.org/rfc-index.xml (`<rfc-entry>`)
+
+The method `RelatonIetf::DataFetcher.fetch(source, output: "data", format: "yaml")` converts all the documents from the dataset and save them to the `./data` folder in YAML format.
+
+Arguments:
+
+- `source` - dataset name (`ietf-rfcsubseries` or `ietf-internet-drafts`)
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml`, `bibxml` (default `yaml`).
+
+For `ietf-rfcsubseries` dataset only special XML format is supported:
+
+[sourse.xml]
+----
+<referencegroup anchor="BCP14" target="https://www.rfc-editor.org/info/bcp14">
+  <xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.2119.xml" />
+  <xi:include href="https://www.rfc-editor.org/refs/bibxml/reference.RFC.8174.xml" />
+</referencegroup>
+----
+
+[source,ruby]
+----
+RelatonIetf::DataFetcher.fetch "ietf-internet-drafts"
+Started at: 2021-12-17 10:23:20 +0100
+Stopped at: 2021-12-17 10:29:19 +0100
+Done in: 360 sec.
+=> nil
+----
+
 == Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/metanorma/relaton-ietf.

data/lib/relaton_ietf/data_fetcher.rb

@@ -0,0 +1,132 @@
+require "rubygems"
+require "rubygems/package"
+require "zlib"
+require "relaton_ietf/rfc_index_entry"
+require "relaton_ietf/rfc_entry"
+
+module RelatonIetf
+  class DataFetcher
+    #
+    # Data fetcher initializer
+    #
+    # @param [String] source source name
+    # @param [String] output directory to save files
+    # @param [String] format format of output files (xml, yaml, bibxml);
+    #   for ietf-rfcsubseries source only: xml
+    #
+    def initialize(source, output, format)
+      @source = source
+      @output = output
+      @format = format
+      @ext = @format.sub(/^bib|^rfc/, "")
+      @files = []
+    end
+
+    #
+    # Initialize fetcher and run fetch
+    #
+    # @param [String] source source name
+    # @param [Strin] output directory to save files, default: "data"
+    # @param [Strin] format format of output files (xml, yaml, bibxml);
+    #   default: yaml; for ietf-rfcsubseries source only: xml
+    #
+    def self.fetch(source, output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(source, output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Fetch documents
+    #
+    def fetch
+      case @source
+      when "ietf-rfcsubseries" then fetch_ieft_rfcsubseries
+      when "ietf-internet-drafts" then fetch_ieft_internet_drafts
+      when "ietf-rfc-entries" then fetch_ieft_rfcs
+      end
+    end
+
+    #
+    # Fetches ietf-rfcsubseries documents
+    #
+    def fetch_ieft_rfcsubseries
+      rfc_index.xpath("xmlns:bcp-entry|xmlns:fyi-entry|xmlns:std-entry").each do |doc|
+        save_doc RfcIndexEntry.parse(doc)
+      end
+    end
+
+    #
+    # Fetches ietf-internet-drafts documents
+    #
+    def fetch_ieft_internet_drafts # rubocop:disable Metrics/MethodLength
+      gz = OpenURI.open_uri("https://www.ietf.org/lib/dt/sprint/bibxml-ids.tgz")
+      z = Zlib::GzipReader.new(gz)
+      io = StringIO.new(z.read)
+      z.close
+      Gem::Package::TarReader.new io do |tar|
+        tar.each do |tarfile|
+          next if tarfile.directory?
+
+          save_doc RelatonBib::BibXMLParser.parse(tarfile.read)
+        end
+      end
+    end
+
+    def fetch_ieft_rfcs
+      rfc_index.xpath("xmlns:rfc-entry").each do |doc|
+        save_doc RfcEntry.parse(doc)
+      end
+    end
+
+    def rfc_index
+      uri = URI "https://www.rfc-editor.org/rfc-index.xml"
+      Nokogiri::XML(Net::HTTP.get(uri)).at("/xmlns:rfc-index")
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonIetf::RfcIndexEntry, nil] rfc index entry
+    #
+    def save_doc(entry) # rubocop:disable Metrics/MethodLength
+      return unless entry
+
+      c = case @format
+          when "xml" then entry.to_xml(bibdata: true)
+          when "yaml" then entry.to_hash.to_yaml
+          else entry.send("to_#{@format}")
+          end
+      file = file_name entry
+      if @files.include? file
+        warn "File #{file} already exists. Document: #{entry.docnumber}"
+      else
+        @files << file
+      end
+      File.write file, c, encoding: "UTF-8"
+    end
+
+    #
+    # Generate file name
+    #
+    # @param [RelatonIetf::RfcIndexEntry] entry
+    #
+    # @return [String] file name
+    #
+    def file_name(entry)
+      id = if entry.respond_to? :docidentifier
+             entry.docidentifier.detect { |i| i.type == "Internet-Draft" }&.id
+           end
+      id ||= entry.docnumber
+      if @source == "ietf-internet-drafts" then id.downcase!
+      else id.upcase!
+      end
+      name = id.gsub(/[\s,:\/]/, "_").squeeze("_")
+      File.join @output, "#{name}.#{@ext}"
+    end
+  end
+end

data/lib/relaton_ietf/processor.rb

@@ -3,11 +3,12 @@ require "relaton_ietf/xml_parser"
 
 module RelatonIetf
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_ietf
       @prefix = "IETF"
-      @defaultprefix = /^RFC /
+      @defaultprefix = /^(IETF|RFC|BCP)\s/
       @idtype = "IETF"
+      @datasets = %w[ietf-rfcsubseries ietf-internet-drafts ietf-rfc-entries]
     end
 
     # @param code [String]
@@ -18,6 +19,18 @@ module RelatonIetf
       ::RelatonIetf::IetfBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from https://www.rfc-editor.org/rfc-index.xml
+    #
+    # @param [String] source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(source, opts)
+      DataFetcher.fetch(source, **opts)
+    end
+
     # @param xml [String]
     # @return [RelatonIetf::IetfBibliographicItem]
     def from_xml(xml)

data/lib/relaton_ietf/rfc_entry.rb

@@ -0,0 +1,186 @@
+module RelatonIetf
+  class RfcEntry
+    #
+    # Initalize parser
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    def initialize(doc)
+      @doc = doc
+    end
+
+    #
+    # Initialize parser & parse document
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    # @return [RelatonIetf::IetfBibliographicItem] bib item
+    #
+    def self.parse(doc)
+      new(doc).parse
+    end
+
+    #
+    # Parse document
+    #
+    # @return [RelatonIetf::IetfBibliographicItem] bib item
+    #
+    def parse # rubocop:disable Metrics/MethodLength
+      IetfBibliographicItem.new(
+        type: "standard",
+        language: ["en"],
+        script: ["Latn"],
+        fetched: Date.today.to_s,
+        docid: parse_docid,
+        docnumber: code,
+        title: parse_title,
+        link: parse_link,
+        date: parse_date,
+        contributor: parse_contributor,
+        keyword: parse_keyword,
+        abstract: parse_abstract,
+        relation: parse_relation,
+        status: parse_status,
+        editorialgroup: parse_editorialgroup,
+      )
+    end
+
+    #
+    # Parse document identifiers
+    #
+    # @return [Array<RelatonBib::DocumentIdettifier>] document identifiers
+    #
+    def parse_docid
+      ids = [RelatonBib::DocumentIdentifier.new(id: pub_id, type: "IETF")]
+      doi = @doc.at("./xmlns:doi").text
+      ids << RelatonBib::DocumentIdentifier.new(id: doi, type: "DOI")
+      ids
+    end
+
+    #
+    # Parse document title
+    #
+    # @return [Array<RelatonBib::TypedTileString>] document title
+    #
+    def parse_title
+      content = @doc.at("./xmlns:title").text
+      [RelatonBib::TypedTitleString.new(content: content, type: "main")]
+    end
+
+    #
+    # Create PubID
+    #
+    # @return [String] PubID
+    #
+    def pub_id
+      "IETF #{code.sub(/^(RFC)(\d+)/, '\1 \2')}"
+    end
+
+    #
+    # Parse document code
+    #
+    # @return [String] document code
+    #
+    def code
+      @doc.at("./xmlns:doc-id").text
+    end
+
+    #
+    # Create link
+    #
+    # @return [Array<RelatonBib::TypedUri>] 
+    #
+    def parse_link
+      num = code[-4..-1].sub(/^0+/, "")
+      url = "https://www.rfc-editor.org/info/rfc#{num}"
+      [RelatonBib::TypedUri.new(content: url, type: "src")]
+    end
+
+    #
+    # Parse document date
+    #
+    # @return [Array<RelatonBib::BibliographicDate>] document date
+    #
+    def parse_date
+      @doc.xpath("./xmlns:date").map do |date|
+        month = date.at("./xmlns:month").text
+        year = date.at("./xmlns:year").text
+        on = "#{year}-#{Date::MONTHNAMES.index(month).to_s.rjust(2, '0')}"
+        RelatonBib::BibliographicDate.new(on: on, type: "published")
+      end
+    end
+
+    #
+    # Parse document contributors
+    #
+    # @return [Array<RelatonBib::ContributionInfo>] document contributors
+    #
+    def parse_contributor
+      @doc.xpath("./xmlns:author").map do |contributor|
+        n = contributor.at("./xmlns:name").text
+        name = RelatonBib::LocalizedString.new( n,  "en",  "Latn")
+        fname = RelatonBib::FullName.new(completename: name)
+        person = RelatonBib::Person.new(name: fname)
+        RelatonBib::ContributionInfo.new(entity: person, role: [{ type: "author" }])
+      end
+    end
+
+    #
+    # Parse document keywords
+    #
+    # @return [Array<String>] document keywords
+    #
+    def parse_keyword
+      @doc.xpath("./xmlns:keywords/xmlns:kw").map &:text
+    end
+
+    #
+    # Parse document abstract
+    #
+    # @return [Array<RelatonBib::FormattedString>] document abstract
+    #
+    def parse_abstract
+      @doc.xpath("./xmlns:abstract").map do |c|
+        RelatonBib::FormattedString.new(content: c.text, language: "en",
+                                        script: "Latn", format: "text/html")
+      end
+    end
+
+    #
+    # Parse document relations
+    #
+    # @return [Arra<RelatonBib::DocumentRelation>] document relations
+    #
+    def parse_relation
+      types = { "updates" => "updates", "obsoleted-by" => "obsoletedBy"}
+      @doc.xpath("./xmlns:updates/xmlns:doc-id|./xmlns:obsoleted-by/xmlns:doc-id").map do |r|
+        fref = RelatonBib::FormattedRef.new(content: r.text)
+        bib = IetfBibliographicItem.new(formattedref: fref)
+        RelatonBib::DocumentRelation.new(type: types[r.parent.name], bibitem: bib)
+      end
+    end
+
+    #
+    # Parse document status
+    #
+    # @return [RelatonBib::DocuemntStatus] document status
+    #
+    def parse_status
+      stage = @doc.at("./xmlns:current-status").text
+      RelatonBib::DocumentStatus.new(stage: stage)
+    end
+
+    #
+    # Parse document editorial group
+    #
+    # @return [RelatonBib::EditorialGroup] document editorial group
+    #
+    def parse_editorialgroup
+      tc = @doc.xpath("./xmlns:wg_acronym").map do |wg|
+        wg = RelatonBib::WorkGroup.new(name: wg.text)
+        RelatonBib::TechnicalCommittee.new(wg)
+      end
+      RelatonBib::EditorialGroup.new(tc)
+    end
+  end
+end

data/lib/relaton_ietf/rfc_index_entry.rb

@@ -0,0 +1,87 @@
+module RelatonIetf
+  class RfcIndexEntry
+    #
+    # Document parser initalization
+    #
+    # @param [String] name document type name
+    # @param [String] doc_id document id
+    # @param [Array<String>] is_also included document ids
+    #
+    def initialize(name, doc_id, is_also)
+      @name = name
+      @shortnum = doc_id[-4..-1].sub(/^0+/, "")
+      @doc_id = doc_id
+      @is_also = is_also
+    end
+
+    #
+    # Initialize document parser and run it
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    #
+    # @return [RelatonIetf:RfcIndexEntry, nil]
+    #
+    def self.parse(doc)
+      doc_id = doc.at("./xmlns:doc-id")
+      is_also = doc.xpath("./xmlns:is-also/xmlns:doc-id").map &:text
+      return unless doc_id && is_also.any?
+
+      name = doc.name.split("-").first
+      new(name, doc_id.text, is_also).parse
+    end
+
+    def parse
+      IetfBibliographicItem.new(
+        docnumber: docnumber,
+        type: "standard",
+        docid: parse_docid,
+        language: ["en"],
+        script: ["Latn"],
+        link: parse_link,
+        formattedref: formattedref,
+        relation: parse_relation,
+      )
+    end
+
+    #
+    # Document id
+    #
+    # @return [Strinng] document id
+    #
+    def docnumber
+      @doc_id
+    end
+
+    def parse_docid
+      [
+        RelatonBib::DocumentIdentifier.new(type: "IETF", id: pub_id),
+        RelatonBib::DocumentIdentifier.new(type: "rfc-anchor", id: anchor),
+      ]
+    end
+
+    def pub_id
+      "IETF #{@name.upcase} #{@shortnum}"
+    end
+
+    def anchor
+      "#{@name.upcase}#{@shortnum}"
+    end
+
+    def parse_link
+      [RelatonBib::TypedUri.new(type: "src", content: "https://www.rfc-editor.org/info/#{@name}#{@shortnum}")]
+    end
+
+    def formattedref
+      RelatonBib::FormattedRef.new(
+        content: anchor, language: "en", script: "Latn",
+      )
+    end
+
+    def parse_relation
+      @is_also.map do |ref|
+        bib = IetfBibliography.get ref.sub(/^(RFC)(\d+)/, '\1 \2')
+        { type: "includes", bibitem: bib }
+      end
+    end
+  end
+end

data/lib/relaton_ietf/version.rb

@@ -1,3 +1,3 @@
 module RelatonIetf
-  VERSION = "1.9.2".freeze
+  VERSION = "1.9.6".freeze
 end

data/lib/relaton_ietf.rb

@@ -6,6 +6,7 @@ require "relaton_ietf/ietf_bibliographic_item"
 require "relaton_ietf/xml_parser"
 require "relaton_ietf/hash_converter"
 require "relaton_ietf/committee"
+require "relaton_ietf/data_fetcher"
 
 require "relaton/provider_ietf"
 

data/relaton_ietf.gemspec

@@ -38,5 +38,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 
-  spec.add_dependency "relaton-bib", ">= 1.9.5"
+  spec.add_dependency "relaton-bib", ">= 1.9.8"
+  spec.add_dependency "zlib", "~> 1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-ietf
 version: !ruby/object:Gem::Version
-  version: 1.9.2
+  version: 1.9.6
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-26 00:00:00.000000000 Z
+date: 2021-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -128,14 +128,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.5
+        version: 1.9.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.5
+        version: 1.9.8
+- !ruby/object:Gem::Dependency
+  name: zlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
 description: "RelatonIetf: retrieve IETF Standards for bibliographic use \nusing the
   BibliographicItem model.\n\nFormerly known as rfcbib.\n"
 email:
@@ -164,10 +178,13 @@ files:
 - lib/relaton/provider_ietf.rb
 - lib/relaton_ietf.rb
 - lib/relaton_ietf/committee.rb
+- lib/relaton_ietf/data_fetcher.rb
 - lib/relaton_ietf/hash_converter.rb
 - lib/relaton_ietf/ietf_bibliographic_item.rb
 - lib/relaton_ietf/ietf_bibliography.rb
 - lib/relaton_ietf/processor.rb
+- lib/relaton_ietf/rfc_entry.rb
+- lib/relaton_ietf/rfc_index_entry.rb
 - lib/relaton_ietf/scrapper.rb
 - lib/relaton_ietf/version.rb
 - lib/relaton_ietf/xml_parser.rb