relaton-ieee 1.9.0 → 1.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc70675477b3d216a6e596b63c89e2fd867cb725372226d45fdd67da6d1f170e
-  data.tar.gz: 0e24be2cab28dd83370e90e19fb3058432005a475f299c0d79eb216177a0054b
+  metadata.gz: 53ecd27da5566b505d1ced7fc6f29af23ec9a45124501e790596350851c362a0
+  data.tar.gz: 2e6b02cedacffd9d44004aa480d57037abfcd6ddf9a5371c5fed0bea2aa533f3
 SHA512:
-  metadata.gz: 05e9abb808f6517cfd8dd9f85b0b033594a8bbfb7c179f18e5db493c9b3afac4d9fb9adf902b32a5d7444791770894e449341c2cbfd4380a254cf232b01e41ac
-  data.tar.gz: db0577970220cfcb39ca1b00083851a7d49630e61870d8e3cd2de330ac8ea5d8fe14c6c9afd6640662b9e04e05428acec4bda234b34f2f02527b32b125a62c7a
+  metadata.gz: fb1c5ed0240d28a7494e0e7f7c587ad29610f1ec83aa6ca3f390b9d98256c96b2d8e735d3e46425c7d0b3457b66a1e020c0744d7dfa0b8c031535c866e0a53a7
+  data.tar.gz: ba00b7b538a63696c5e646953eb4f68a76e8c25dd7e1a1a221e2c8edacf1155c8f0eb14951421b253e252190d0f728f681f25712bc5798f6235e93463f97aaa8

data/.gitignore

@@ -11,4 +11,5 @@
 .rspec_status
 .rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
 .vscode/
+ieee-rawbib/
 Gemfile.lock

data/README.adoc

@@ -158,6 +158,25 @@ hash = YAML.load_file 'spec/fixtures/ieee_528_2019.yaml'
  ...
 ----
 
+=== Fetch data
+
+There is an IEEE dataset https://github.com/ietf-ribose/ieee-rawbib which can be converted into BibXML/BibYAML formats. The dataset needs to be placed into local directiory.
+
+The method `RelatonIeee::DataFetcher.fetch(output: "data", format: "yaml")` converts all the documents from the local `ieee-rawbib` directory and save them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonIeee::DataFetcher.fetch
+Started at: 2021-09-24 17:55:07 +0200
+Stopped at: 2021-09-24 17:57:30 +0200
+Done in: 143 sec.
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/relaton_ieee/data_fetcher.rb

@@ -0,0 +1,206 @@
+require "zip"
+require "relaton_ieee/data_parser"
+
+module RelatonIeee
+  class DataFetcher
+    RELATION_TYPES = {
+      "S" => { type: "obsoletedBy" },
+      "V" => { type: "updates", description: "revises" },
+      "T" => { type: "updates", description: "amends" },
+      "C" => { type: "updates", description: "corrects" },
+      "O" => { type: "adoptedFrom" },
+      "P" => { type: "complementOf", description: "supplement" },
+      "N" => false, "G" => false,
+      "F" => false, "I" => false,
+      "E" => false, "B" => false, "W" => false
+    }.freeze
+
+    # @return [Hash] list of AMSID => PubID
+    attr_reader :backrefs
+
+    #
+    # Create RelatonIeee::DataFetcher instance
+    #
+    # @param [String] output output dir
+    # @param [Strong] format output format. Allowed values: "yaml" or "xml"
+    #
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @crossrefs = {}
+      @backrefs = {}
+    end
+
+    #
+    # Convert documents from `ieee-rawbib` dir (IEEE dataset) to BibYAML/BibXML
+    #
+    # @param [String] output ('data') output dir
+    # @param [String] format ('yaml') output format.
+    #   Allowed values: "yaml" or "xml"
+    #
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Convert documents from `ieee-rawbib` dir (IEEE dataset) to BibYAML/BibXML
+    #
+    def fetch # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      Dir["ieee-rawbib/**/*.{xml,zip}"].reject { |f| f["Deleted_"] }.each do |f|
+        xml = case File.extname(f)
+              when ".zip" then read_zip f
+              when ".xml" then File.read f, encoding: "UTF-8"
+              end
+        fetch_doc xml, f
+      rescue StandardError => e
+        warn "File: #{f}"
+        warn e.message
+        warn e.backtrace
+      end
+      update_relations
+    end
+
+    #
+    # Extract XML file from zip archive
+    #
+    # @param [String] file path to achive
+    #
+    # @return [String] file content
+    #
+    def read_zip(file)
+      Zip::File.open(file) do |zf|
+        entry = zf.glob("**/*.xml").first
+        entry.get_input_stream.read
+      end
+    end
+
+    #
+    # Parse document and save it
+    #
+    # @param [String] xml content
+    # @param [String] filename source file
+    #
+    def fetch_doc(xml, filename) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      doc = Nokogiri::XML(xml).at("/publication")
+      unless doc
+        warn "Empty file: #{filename}"
+        return
+      end
+      bib = DataParser.parse doc, self
+      amsid = doc.at("./publicationinfo/amsid").text
+      if backrefs.value?(bib.docidentifier[0].id) && /updates\.\d+/ !~ filename
+        oamsid = backrefs.key bib.docidentifier[0].id
+        warn "Document exists ID: \"#{bib.docidentifier[0].id}\" AMSID: "\
+             "\"#{amsid}\" source: \"#{filename}\". Other AMSID: \"#{oamsid}\""
+        if bib.docidentifier[0].id.include?(bib.docnumber)
+          save_doc bib # rewrite file if the PubID mathces to the docnumber
+          backrefs[amsid] = bib.docidentifier[0].id
+        end
+      else
+        save_doc bib
+        backrefs[amsid] = bib.docidentifier[0].id
+      end
+    end
+
+    #
+    # Save unresolved relation reference
+    #
+    # @param [String] docnumber of main document
+    # @param [Nokogiri::XML::Element] amsid relation data
+    #
+    def add_crossref(docnumber, amsid)
+      return if RELATION_TYPES[amsid[:type]] == false
+
+      ref = { amsid: amsid.text, type: amsid[:type] }
+      if @crossrefs[docnumber]
+        @crossrefs[docnumber] << ref
+      else @crossrefs[docnumber] = [ref]
+      end
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonIeee::IeeeBibliographicItem] bib
+    #
+    def save_doc(bib)
+      c = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
+      File.write file_name(bib.docnumber), c, encoding: "UTF-8"
+    end
+
+    #
+    # Make filename from PubID
+    #
+    # @param [String] docnumber
+    #
+    # @return [String] filename
+    #
+    def file_name(docnumber)
+      name = docnumber.gsub(/[.\s,:\/]/, "_").squeeze("_").upcase
+      File.join @output, "#{name}.#{@format}"
+    end
+
+    #
+    # Update unresoverd relations
+    #
+    def update_relations # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      @crossrefs.each do |dnum, rfs|
+        bib = nil
+        rfs.each do |rf|
+          if backrefs[rf[:amsid]]
+            rel = create_relation(rf[:type], backrefs[rf[:amsid]])
+            if rel
+              bib ||= read_bib(dnum)
+              bib.relation << rel
+              save_doc bib
+            end
+          else
+            warn "Unresolved relation: '#{rf[:amsid]}' type: '#{rf[:type]}' for '#{dnum}'"
+          end
+        end
+      end
+    end
+
+    #
+    # Create relation instance
+    #
+    # @param [String] type IEEE relation type
+    # @param [String] fref reference
+    #
+    # @return [RelatonBib::DocumentRelation]
+    #
+    def create_relation(type, fref)
+      return if RELATION_TYPES[type] == false
+
+      fr = RelatonBib::FormattedRef.new(content: fref)
+      bib = IeeeBibliographicItem.new formattedref: fr
+      desc = RELATION_TYPES[type][:description]
+      description = desc && RelatonBib::FormattedString.new(content: desc, language: "en", script: "Latn")
+      RelatonBib::DocumentRelation.new(
+        type: RELATION_TYPES[type][:type],
+        description: description,
+        bibitem: bib,
+      )
+    end
+
+    #
+    # Read document form BibXML/BibYAML file
+    #
+    # @param [String] docnumber
+    #
+    # @return [RelatonIeee::IeeeBibliographicItem]
+    #
+    def read_bib(docnumber)
+      c = File.read file_name(docnumber), encoding: "UTF-8"
+      if @format == "xml" then XMLParser.from_xml c
+      else IeeeBibliographicItem.from_hash YAML.safe_load(c)
+      end
+    end
+  end
+end

data/lib/relaton_ieee/data_parser.rb

@@ -0,0 +1,266 @@
+module RelatonIeee
+  class DataParser
+    DATETYPES = { "OriginalPub" => "created", "ePub" => "published",
+                  "LastInspecUpd" => "updated" }.freeze
+
+    attr_reader :doc, :fetcher
+
+    #
+    # Create RelatonIeee::DataParser instance
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    # @param [RelatonIeee::DataFetcher] fetcher
+    #
+    def initialize(doc, fetcher)
+      @doc = doc
+      @fetcher = fetcher
+    end
+
+    #
+    # Parse IEEE document
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    # @param [RelatonIeee::DataFetcher] fetcher <description>
+    #
+    # @return [RelatonIeee::IeeeBibliographicItem]
+    #
+    def self.parse(doc, fetcher)
+      new(doc, fetcher).parse
+    end
+
+    #
+    # Parse IEEE document
+    #
+    # @return [RelatonIeee::IeeeBibliographicItem]
+    #
+    def parse # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+      args = {
+        type: "standard",
+        docnumber: docnumber,
+        title: parse_title,
+        date: parse_date,
+        docid: parse_docid,
+        contributor: parse_contributor,
+        abstract: parse_abstract,
+        copyright: parse_copyright,
+        language: ["en"],
+        script: ["Latn"],
+        status: parse_status,
+        relation: parse_relation,
+        link: parse_link,
+        keyword: parse_keyword,
+        ics: parse_ics,
+      }
+      IeeeBibliographicItem.new(**args)
+    end
+
+    #
+    # Parse title
+    #
+    # @return [RelatonBib::TypedTitleStringCollection]
+    #
+    def parse_title
+      t = doc.at("./volume/article/title").text
+      RelatonBib::TypedTitleString.from_string t
+    end
+
+    #
+    # Parse date
+    #
+    # @return [Array<RelatonBib::BibliographicDate>]
+    #
+    def parse_date # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength
+      dates = doc.xpath("./volume/article/articleinfo/date").map do |d|
+        da = [d.at("./year").text]
+        m = d.at("./month")&.text
+        if m
+          month = Date::ABBR_MONTHNAMES.index(m.sub(/\./, "")) || m
+          da << month.to_s.rjust(2, "0")
+        end
+        day = d.at("./day")
+        da << day.text.rjust(2, "0") if day
+        on = da.compact.join "-"
+        RelatonBib::BibliographicDate.new type: DATETYPES[d[:datetype]], on: on
+      end
+      pad = doc.at("./publicationinfo/PubApprovalDate")
+      if pad
+        issued = parse_date_string pad.text
+        dates << RelatonBib::BibliographicDate.new(type: "issued", on: issued)
+      end
+      dates
+    end
+
+    #
+    # Convert date string with month name to numeric date
+    #
+    # @param [String] date source date
+    #
+    # @return [String] numeric date
+    #
+    def parse_date_string(date)
+      case date
+      when /^\d{4}$/ then date
+      when /^\d{1,2}\s\w+\.?\s\d{4}/ then Date.parse(date).to_s
+      end
+    end
+
+    #
+    # Parse identifiers
+    #
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    #
+    def parse_docid
+      ids = [{ id: doc.at("./title").text, type: "IEEE" }]
+      isbn = doc.at("./publicationinfo/isbn")
+      ids << { id: isbn.text, type: "ISBN" } if isbn
+      doi = doc.at("./volume/article/articleinfo/articledoi")
+      ids << { id: doi.text, type: "DOI" } if doi
+      ids.map do |dcid|
+        RelatonBib::DocumentIdentifier.new(**dcid)
+      end
+    end
+
+    #
+    # Parse docnumber
+    #
+    # @return [String] PubID
+    #
+    def docnumber
+      @docnumber ||= doc.at("./publicationinfo/stdnumber").text
+    end
+
+    #
+    # Parse contributors
+    #
+    # @return [Array<RelatonBib::ContributionInfo>]
+    #
+    def parse_contributor # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      doc.xpath("./publicationinfo/publisher").map do |contrib|
+        n = contrib.at("./publishername").text
+        addr = contrib.xpath("./address").map do |a|
+          RelatonBib::Address.new(
+            street: [],
+            city: a.at("./city")&.text,
+            country: a.at("./country").text,
+          )
+        end
+        e = create_org n, addr
+        RelatonBib::ContributionInfo.new entity: e, role: [type: "publisher"]
+      end
+    end
+
+    #
+    # Create organization
+    #
+    # @param [String] name organization's name
+    # @param [Array<Hash>] addr address
+    #
+    # @return [RelatonBib::Organization]
+    def create_org(name, addr = []) # rubocop:disable Metrics/MethodLength
+      case name
+      when "IEEE"
+        abbr = name
+        n = "Institute of Electrical and Electronics Engineers"
+        url = "http://www.ieee.org"
+      when "ANSI"
+        abbr = name
+        n = "American National Standards Institute"
+        url = "https://www.ansi.org"
+      else n = name
+      end
+      RelatonBib::Organization.new(
+        name: n, abbreviation: abbr, url: url, contact: addr,
+      )
+    end
+
+    #
+    # Parse abstract
+    #
+    # @return [Array<RelatonBib::FormattedString>]
+    #
+    def parse_abstract
+      doc.xpath("./volume/article/articleinfo/abstract").map do |a|
+        RelatonBib::FormattedString.new(
+          content: a.text, language: "en", script: "Latn",
+        )
+      end
+    end
+
+    #
+    # Parse copyright
+    #
+    # @return [Array<RelatonBib::CopyrightAssociation>]
+    #
+    def parse_copyright
+      doc.xpath("./publicationinfo/copyrightgroup/copyright").map do |c|
+        owner = c.at("./holder").text.split("/").map do |own|
+          RelatonBib::ContributionInfo.new entity: create_org(own)
+        end
+        RelatonBib::CopyrightAssociation.new(
+          owner: owner, from: c.at("./year").text,
+        )
+      end
+    end
+
+    #
+    # Parse status
+    #
+    # @return [RelatonBib::DocumentStatus]
+    #
+    def parse_status
+      stage = doc.at("./publicationinfo/standard_status").text
+      RelatonBib::DocumentStatus.new stage: stage
+    end
+
+    #
+    # Parse relation
+    #
+    # @return [RelatonBib::DocRelationCollection]
+    #
+    def parse_relation # rubocop:disable Metrics/AbcSize
+      rels = []
+      doc.xpath("./publicationinfo/standard_relationship").each do |r|
+        if (ref = fetcher.backrefs[r.text])
+          rel = fetcher.create_relation(r[:type], ref)
+          rels << rel if rel
+        elsif !/Inactive Date/.match?(r) then fetcher.add_crossref(docnumber, r)
+        end
+      end
+      RelatonBib::DocRelationCollection.new rels
+    end
+
+    #
+    # Parce link
+    #
+    # @return [Array<RelatonBib::TypedUri>]
+    #
+    def parse_link
+      doc.xpath("./volume/article/articleinfo/amsid").map do |id|
+        l = "https://ieeexplore.ieee.org/document/#{id.text}"
+        RelatonBib::TypedUri.new content: l, type: "src"
+      end
+    end
+
+    #
+    # Parse keyword
+    #
+    # @return [Array<Strign>]
+    #
+    def parse_keyword
+      doc.xpath(
+        "./volume/article/articleinfo/keywordset/keyword/keywordterm",
+      ).map &:text
+    end
+
+    #
+    # Parse ICS
+    #
+    # @return [Array<RelatonBib::ICS>]
+    #
+    def parse_ics
+      doc.xpath("./publicationinfo/icscodes/code_term").map do |ics|
+        RelatonBib::ICS.new code: ics[:codenum], text: ics.text
+      end
+    end
+  end
+end

data/lib/relaton_ieee/ieee_bibliographic_item.rb

@@ -5,7 +5,7 @@ module RelatonIeee
 
     # @param committee [Array<RelatonIeee::Committee>]
     def initialize(**args)
-      @committee = args.delete :committee
+      @committee = args.delete(:committee) || []
       super
     end
 
@@ -13,7 +13,7 @@ module RelatonIeee
     # @return [RelatonIeee::IeeeBibliographicItem]
     def self.from_hash(hash)
       item_hash = ::RelatonIeee::HashConverter.hash_to_bib(hash)
-      new **item_hash
+      new(**item_hash)
     end
 
     # @param opts [Hash]
@@ -22,7 +22,7 @@ module RelatonIeee
     # @option opts [String] :lang language
     # @return [String] XML
     def to_xml(**opts)
-      super **opts do |bldr|
+      super(**opts) do |bldr|
         if opts[:bibdata] && committee.any?
           bldr.ext do |b|
             committee.each { |c| c.to_xml b }
@@ -34,7 +34,7 @@ module RelatonIeee
     # @return [Hash]
     def to_hash
       hash = super
-      hash["committee"] = committee.map &:to_hash
+      hash["committee"] = committee.map &:to_hash if committee.any?
       hash
     end
 

data/lib/relaton_ieee/ieee_bibliography.rb

@@ -19,7 +19,7 @@ module RelatonIeee
         warn "[relaton-ieee] (\"#{code}\") fetching..."
         result = search(code) || (return nil)
         year ||= code.match(/(?<=-)\d{4}/)&.to_s
-        ret = bib_results_filter(result, year)
+        ret = bib_results_filter(result, code, year)
         if ret[:ret]
           item = ret[:ret].fetch
           warn "[relaton-ieee] (\"#{code}\") found #{item.docidentifier.first.id}"
@@ -42,9 +42,13 @@ module RelatonIeee
       # @param opts [Hash] options
       #
       # @return [Hash]
-      def bib_results_filter(result, year)
+      def bib_results_filter(result, ref, year)
+        rp1 = ref_parts ref
         missed_years = []
         result.each do |hit|
+          rp2 = ref_parts hit.hit["recordTitle"]
+          next if rp1[:code] != rp2[:code] || rp1[:corr] != rp2[:corr]
+
           return { ret: hit } if !year
 
           return { ret: hit } if year.to_i == hit.hit[:year]
@@ -54,6 +58,15 @@ module RelatonIeee
         { years: missed_years.uniq }
       end
 
+      def ref_parts(ref)
+        %r{
+          ^(?:IEEE\s(?:Std\s)?)?
+          (?<code>[^-/]+)
+          (?:-(?<year>\d{4}))?
+          (?:/(?<corr>\w+\s\d+-\d{4}))?
+        }x.match ref
+      end
+
       # @param code [Strig]
       # @param year [String]
       # @param missed_years [Array<Strig>]

data/lib/relaton_ieee/processor.rb

@@ -4,11 +4,12 @@ module RelatonIeee
   class Processor < Relaton::Processor
     attr_reader :idtype
 
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_ieee
       @prefix = "IEEE"
       @defaultprefix = %r{^IEEE\s}
       @idtype = "IEEE"
+      @datasets = %w[ieee-rawbib]
     end
 
     # @param code [String]
@@ -19,6 +20,18 @@ module RelatonIeee
       ::RelatonIeee::IeeeBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from ./iee-rawbib directory
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonIeee::IeeeBibliographicItem]
     def from_xml(xml)

data/lib/relaton_ieee/version.rb

@@ -1,3 +1,3 @@
 module RelatonIeee
-  VERSION = "1.9.0".freeze
+  VERSION = "1.9.1".freeze
 end

data/lib/relaton_ieee.rb

@@ -10,6 +10,7 @@ require "relaton_ieee/ieee_bibliographic_item"
 require "relaton_ieee/committee"
 require "relaton_ieee/xml_parser"
 require "relaton_ieee/hash_converter"
+require "relaton_ieee/data_fetcher"
 
 module RelatonIeee
   class Error < StandardError; end

data/relaton_ieee.gemspec

@@ -39,4 +39,5 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "faraday", "~> 1.1"
   spec.add_dependency "relaton-bib", "~> 1.9.0"
+  spec.add_dependency "rubyzip", "~> 2.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-ieee
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.9.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-26 00:00:00.000000000 Z
+date: 2021-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.9.0
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.0
 description: 'RelatonIeee: retrieve IEEE Standards for bibliographic use using the
   IeeeBibliographicItem model'
 email:
@@ -133,6 +147,8 @@ files:
 - grammars/reqt.rng
 - lib/relaton_ieee.rb
 - lib/relaton_ieee/committee.rb
+- lib/relaton_ieee/data_fetcher.rb
+- lib/relaton_ieee/data_parser.rb
 - lib/relaton_ieee/hash_converter.rb
 - lib/relaton_ieee/hit.rb
 - lib/relaton_ieee/hit_collection.rb