relaton-ieee 1.7.4 → 1.9.1

data/grammars/reqt.rng

@@ -30,15 +30,34 @@
         <data type="boolean"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="number"/>
+    </optional>
     <optional>
       <attribute name="subsequence"/>
     </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
     <attribute name="id">
       <data type="ID"/>
     </attribute>
     <optional>
       <attribute name="filename"/>
     </optional>
+    <optional>
+      <attribute name="model"/>
+    </optional>
+    <optional>
+      <attribute name="type"/>
+    </optional>
     <optional>
       <ref name="reqtitle"/>
     </optional>
@@ -48,9 +67,9 @@
     <optional>
       <ref name="subject"/>
     </optional>
-    <optional>
+    <zeroOrMore>
       <ref name="reqinherit"/>
-    </optional>
+    </zeroOrMore>
     <zeroOrMore>
       <ref name="classification"/>
     </zeroOrMore>
@@ -135,6 +154,16 @@
         <data type="boolean"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
     <oneOrMore>
       <ref name="BasicBlock"/>
     </oneOrMore>

data/lib/relaton_ieee/data_fetcher.rb

@@ -0,0 +1,206 @@
+require "zip"
+require "relaton_ieee/data_parser"
+
+module RelatonIeee
+  class DataFetcher
+    RELATION_TYPES = {
+      "S" => { type: "obsoletedBy" },
+      "V" => { type: "updates", description: "revises" },
+      "T" => { type: "updates", description: "amends" },
+      "C" => { type: "updates", description: "corrects" },
+      "O" => { type: "adoptedFrom" },
+      "P" => { type: "complementOf", description: "supplement" },
+      "N" => false, "G" => false,
+      "F" => false, "I" => false,
+      "E" => false, "B" => false, "W" => false
+    }.freeze
+
+    # @return [Hash] list of AMSID => PubID
+    attr_reader :backrefs
+
+    #
+    # Create RelatonIeee::DataFetcher instance
+    #
+    # @param [String] output output dir
+    # @param [Strong] format output format. Allowed values: "yaml" or "xml"
+    #
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @crossrefs = {}
+      @backrefs = {}
+    end
+
+    #
+    # Convert documents from `ieee-rawbib` dir (IEEE dataset) to BibYAML/BibXML
+    #
+    # @param [String] output ('data') output dir
+    # @param [String] format ('yaml') output format.
+    #   Allowed values: "yaml" or "xml"
+    #
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Convert documents from `ieee-rawbib` dir (IEEE dataset) to BibYAML/BibXML
+    #
+    def fetch # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      Dir["ieee-rawbib/**/*.{xml,zip}"].reject { |f| f["Deleted_"] }.each do |f|
+        xml = case File.extname(f)
+              when ".zip" then read_zip f
+              when ".xml" then File.read f, encoding: "UTF-8"
+              end
+        fetch_doc xml, f
+      rescue StandardError => e
+        warn "File: #{f}"
+        warn e.message
+        warn e.backtrace
+      end
+      update_relations
+    end
+
+    #
+    # Extract XML file from zip archive
+    #
+    # @param [String] file path to achive
+    #
+    # @return [String] file content
+    #
+    def read_zip(file)
+      Zip::File.open(file) do |zf|
+        entry = zf.glob("**/*.xml").first
+        entry.get_input_stream.read
+      end
+    end
+
+    #
+    # Parse document and save it
+    #
+    # @param [String] xml content
+    # @param [String] filename source file
+    #
+    def fetch_doc(xml, filename) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      doc = Nokogiri::XML(xml).at("/publication")
+      unless doc
+        warn "Empty file: #{filename}"
+        return
+      end
+      bib = DataParser.parse doc, self
+      amsid = doc.at("./publicationinfo/amsid").text
+      if backrefs.value?(bib.docidentifier[0].id) && /updates\.\d+/ !~ filename
+        oamsid = backrefs.key bib.docidentifier[0].id
+        warn "Document exists ID: \"#{bib.docidentifier[0].id}\" AMSID: "\
+             "\"#{amsid}\" source: \"#{filename}\". Other AMSID: \"#{oamsid}\""
+        if bib.docidentifier[0].id.include?(bib.docnumber)
+          save_doc bib # rewrite file if the PubID mathces to the docnumber
+          backrefs[amsid] = bib.docidentifier[0].id
+        end
+      else
+        save_doc bib
+        backrefs[amsid] = bib.docidentifier[0].id
+      end
+    end
+
+    #
+    # Save unresolved relation reference
+    #
+    # @param [String] docnumber of main document
+    # @param [Nokogiri::XML::Element] amsid relation data
+    #
+    def add_crossref(docnumber, amsid)
+      return if RELATION_TYPES[amsid[:type]] == false
+
+      ref = { amsid: amsid.text, type: amsid[:type] }
+      if @crossrefs[docnumber]
+        @crossrefs[docnumber] << ref
+      else @crossrefs[docnumber] = [ref]
+      end
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonIeee::IeeeBibliographicItem] bib
+    #
+    def save_doc(bib)
+      c = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
+      File.write file_name(bib.docnumber), c, encoding: "UTF-8"
+    end
+
+    #
+    # Make filename from PubID
+    #
+    # @param [String] docnumber
+    #
+    # @return [String] filename
+    #
+    def file_name(docnumber)
+      name = docnumber.gsub(/[.\s,:\/]/, "_").squeeze("_").upcase
+      File.join @output, "#{name}.#{@format}"
+    end
+
+    #
+    # Update unresoverd relations
+    #
+    def update_relations # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      @crossrefs.each do |dnum, rfs|
+        bib = nil
+        rfs.each do |rf|
+          if backrefs[rf[:amsid]]
+            rel = create_relation(rf[:type], backrefs[rf[:amsid]])
+            if rel
+              bib ||= read_bib(dnum)
+              bib.relation << rel
+              save_doc bib
+            end
+          else
+            warn "Unresolved relation: '#{rf[:amsid]}' type: '#{rf[:type]}' for '#{dnum}'"
+          end
+        end
+      end
+    end
+
+    #
+    # Create relation instance
+    #
+    # @param [String] type IEEE relation type
+    # @param [String] fref reference
+    #
+    # @return [RelatonBib::DocumentRelation]
+    #
+    def create_relation(type, fref)
+      return if RELATION_TYPES[type] == false
+
+      fr = RelatonBib::FormattedRef.new(content: fref)
+      bib = IeeeBibliographicItem.new formattedref: fr
+      desc = RELATION_TYPES[type][:description]
+      description = desc && RelatonBib::FormattedString.new(content: desc, language: "en", script: "Latn")
+      RelatonBib::DocumentRelation.new(
+        type: RELATION_TYPES[type][:type],
+        description: description,
+        bibitem: bib,
+      )
+    end
+
+    #
+    # Read document form BibXML/BibYAML file
+    #
+    # @param [String] docnumber
+    #
+    # @return [RelatonIeee::IeeeBibliographicItem]
+    #
+    def read_bib(docnumber)
+      c = File.read file_name(docnumber), encoding: "UTF-8"
+      if @format == "xml" then XMLParser.from_xml c
+      else IeeeBibliographicItem.from_hash YAML.safe_load(c)
+      end
+    end
+  end
+end

data/lib/relaton_ieee/data_parser.rb

@@ -0,0 +1,266 @@
+module RelatonIeee
+  class DataParser
+    DATETYPES = { "OriginalPub" => "created", "ePub" => "published",
+                  "LastInspecUpd" => "updated" }.freeze
+
+    attr_reader :doc, :fetcher
+
+    #
+    # Create RelatonIeee::DataParser instance
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    # @param [RelatonIeee::DataFetcher] fetcher
+    #
+    def initialize(doc, fetcher)
+      @doc = doc
+      @fetcher = fetcher
+    end
+
+    #
+    # Parse IEEE document
+    #
+    # @param [Nokogiri::XML::Element] doc document
+    # @param [RelatonIeee::DataFetcher] fetcher <description>
+    #
+    # @return [RelatonIeee::IeeeBibliographicItem]
+    #
+    def self.parse(doc, fetcher)
+      new(doc, fetcher).parse
+    end
+
+    #
+    # Parse IEEE document
+    #
+    # @return [RelatonIeee::IeeeBibliographicItem]
+    #
+    def parse # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+      args = {
+        type: "standard",
+        docnumber: docnumber,
+        title: parse_title,
+        date: parse_date,
+        docid: parse_docid,
+        contributor: parse_contributor,
+        abstract: parse_abstract,
+        copyright: parse_copyright,
+        language: ["en"],
+        script: ["Latn"],
+        status: parse_status,
+        relation: parse_relation,
+        link: parse_link,
+        keyword: parse_keyword,
+        ics: parse_ics,
+      }
+      IeeeBibliographicItem.new(**args)
+    end
+
+    #
+    # Parse title
+    #
+    # @return [RelatonBib::TypedTitleStringCollection]
+    #
+    def parse_title
+      t = doc.at("./volume/article/title").text
+      RelatonBib::TypedTitleString.from_string t
+    end
+
+    #
+    # Parse date
+    #
+    # @return [Array<RelatonBib::BibliographicDate>]
+    #
+    def parse_date # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength
+      dates = doc.xpath("./volume/article/articleinfo/date").map do |d|
+        da = [d.at("./year").text]
+        m = d.at("./month")&.text
+        if m
+          month = Date::ABBR_MONTHNAMES.index(m.sub(/\./, "")) || m
+          da << month.to_s.rjust(2, "0")
+        end
+        day = d.at("./day")
+        da << day.text.rjust(2, "0") if day
+        on = da.compact.join "-"
+        RelatonBib::BibliographicDate.new type: DATETYPES[d[:datetype]], on: on
+      end
+      pad = doc.at("./publicationinfo/PubApprovalDate")
+      if pad
+        issued = parse_date_string pad.text
+        dates << RelatonBib::BibliographicDate.new(type: "issued", on: issued)
+      end
+      dates
+    end
+
+    #
+    # Convert date string with month name to numeric date
+    #
+    # @param [String] date source date
+    #
+    # @return [String] numeric date
+    #
+    def parse_date_string(date)
+      case date
+      when /^\d{4}$/ then date
+      when /^\d{1,2}\s\w+\.?\s\d{4}/ then Date.parse(date).to_s
+      end
+    end
+
+    #
+    # Parse identifiers
+    #
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    #
+    def parse_docid
+      ids = [{ id: doc.at("./title").text, type: "IEEE" }]
+      isbn = doc.at("./publicationinfo/isbn")
+      ids << { id: isbn.text, type: "ISBN" } if isbn
+      doi = doc.at("./volume/article/articleinfo/articledoi")
+      ids << { id: doi.text, type: "DOI" } if doi
+      ids.map do |dcid|
+        RelatonBib::DocumentIdentifier.new(**dcid)
+      end
+    end
+
+    #
+    # Parse docnumber
+    #
+    # @return [String] PubID
+    #
+    def docnumber
+      @docnumber ||= doc.at("./publicationinfo/stdnumber").text
+    end
+
+    #
+    # Parse contributors
+    #
+    # @return [Array<RelatonBib::ContributionInfo>]
+    #
+    def parse_contributor # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      doc.xpath("./publicationinfo/publisher").map do |contrib|
+        n = contrib.at("./publishername").text
+        addr = contrib.xpath("./address").map do |a|
+          RelatonBib::Address.new(
+            street: [],
+            city: a.at("./city")&.text,
+            country: a.at("./country").text,
+          )
+        end
+        e = create_org n, addr
+        RelatonBib::ContributionInfo.new entity: e, role: [type: "publisher"]
+      end
+    end
+
+    #
+    # Create organization
+    #
+    # @param [String] name organization's name
+    # @param [Array<Hash>] addr address
+    #
+    # @return [RelatonBib::Organization]
+    def create_org(name, addr = []) # rubocop:disable Metrics/MethodLength
+      case name
+      when "IEEE"
+        abbr = name
+        n = "Institute of Electrical and Electronics Engineers"
+        url = "http://www.ieee.org"
+      when "ANSI"
+        abbr = name
+        n = "American National Standards Institute"
+        url = "https://www.ansi.org"
+      else n = name
+      end
+      RelatonBib::Organization.new(
+        name: n, abbreviation: abbr, url: url, contact: addr,
+      )
+    end
+
+    #
+    # Parse abstract
+    #
+    # @return [Array<RelatonBib::FormattedString>]
+    #
+    def parse_abstract
+      doc.xpath("./volume/article/articleinfo/abstract").map do |a|
+        RelatonBib::FormattedString.new(
+          content: a.text, language: "en", script: "Latn",
+        )
+      end
+    end
+
+    #
+    # Parse copyright
+    #
+    # @return [Array<RelatonBib::CopyrightAssociation>]
+    #
+    def parse_copyright
+      doc.xpath("./publicationinfo/copyrightgroup/copyright").map do |c|
+        owner = c.at("./holder").text.split("/").map do |own|
+          RelatonBib::ContributionInfo.new entity: create_org(own)
+        end
+        RelatonBib::CopyrightAssociation.new(
+          owner: owner, from: c.at("./year").text,
+        )
+      end
+    end
+
+    #
+    # Parse status
+    #
+    # @return [RelatonBib::DocumentStatus]
+    #
+    def parse_status
+      stage = doc.at("./publicationinfo/standard_status").text
+      RelatonBib::DocumentStatus.new stage: stage
+    end
+
+    #
+    # Parse relation
+    #
+    # @return [RelatonBib::DocRelationCollection]
+    #
+    def parse_relation # rubocop:disable Metrics/AbcSize
+      rels = []
+      doc.xpath("./publicationinfo/standard_relationship").each do |r|
+        if (ref = fetcher.backrefs[r.text])
+          rel = fetcher.create_relation(r[:type], ref)
+          rels << rel if rel
+        elsif !/Inactive Date/.match?(r) then fetcher.add_crossref(docnumber, r)
+        end
+      end
+      RelatonBib::DocRelationCollection.new rels
+    end
+
+    #
+    # Parce link
+    #
+    # @return [Array<RelatonBib::TypedUri>]
+    #
+    def parse_link
+      doc.xpath("./volume/article/articleinfo/amsid").map do |id|
+        l = "https://ieeexplore.ieee.org/document/#{id.text}"
+        RelatonBib::TypedUri.new content: l, type: "src"
+      end
+    end
+
+    #
+    # Parse keyword
+    #
+    # @return [Array<Strign>]
+    #
+    def parse_keyword
+      doc.xpath(
+        "./volume/article/articleinfo/keywordset/keyword/keywordterm",
+      ).map &:text
+    end
+
+    #
+    # Parse ICS
+    #
+    # @return [Array<RelatonBib::ICS>]
+    #
+    def parse_ics
+      doc.xpath("./publicationinfo/icscodes/code_term").map do |ics|
+        RelatonBib::ICS.new code: ics[:codenum], text: ics.text
+      end
+    end
+  end
+end

data/lib/relaton_ieee/hit_collection.rb

@@ -15,7 +15,7 @@ module RelatonIeee
     # @param opts [Hash]
     def initialize(ref) # rubocop:disable Metrics/MethodLength
       super
-      code = ref.sub /^IEEE\s(Std\s)?/, ""
+      code = ref.sub(/^IEEE\s(Std\s)?/, "")
       search = CGI.escape({ data: { searchTerm: code } }.to_json)
       url = "#{DOMAIN}/bin/standards/search?data=#{search}"
       resp = Faraday.get url
@@ -23,7 +23,7 @@ module RelatonIeee
       json = JSON.parse resp_json["message"]
       @array = json["response"]["searchResults"]["resultsMapList"]
         .reduce([]) do |s, hit|
-          /^(?:\w+\s)?(?<id>[^-\/]+)(-(?<year>\d{4}))?/ =~ hit["record"]["recordTitle"]
+          /^(?:\w+\s)?(?<id>[A-Z\d.]+)(?:-(?<year>\d{4}))?/ =~ hit["record"]["recordTitle"]
           next s unless id && code =~ %r{^#{id}}
 
           s << Hit.new(hit["record"].merge(code: id, year: year.to_i), self)

data/lib/relaton_ieee/ieee_bibliographic_item.rb

@@ -5,7 +5,7 @@ module RelatonIeee
 
     # @param committee [Array<RelatonIeee::Committee>]
     def initialize(**args)
-      @committee = args.delete :committee
+      @committee = args.delete(:committee) || []
       super
     end
 
@@ -13,7 +13,7 @@ module RelatonIeee
     # @return [RelatonIeee::IeeeBibliographicItem]
     def self.from_hash(hash)
       item_hash = ::RelatonIeee::HashConverter.hash_to_bib(hash)
-      new **item_hash
+      new(**item_hash)
     end
 
     # @param opts [Hash]
@@ -22,7 +22,7 @@ module RelatonIeee
     # @option opts [String] :lang language
     # @return [String] XML
     def to_xml(**opts)
-      super **opts do |bldr|
+      super(**opts) do |bldr|
         if opts[:bibdata] && committee.any?
           bldr.ext do |b|
             committee.each { |c| c.to_xml b }
@@ -34,7 +34,7 @@ module RelatonIeee
     # @return [Hash]
     def to_hash
       hash = super
-      hash["committee"] = committee.map &:to_hash
+      hash["committee"] = committee.map &:to_hash if committee.any?
       hash
     end
 

data/lib/relaton_ieee/ieee_bibliography.rb

@@ -19,7 +19,7 @@ module RelatonIeee
         warn "[relaton-ieee] (\"#{code}\") fetching..."
         result = search(code) || (return nil)
         year ||= code.match(/(?<=-)\d{4}/)&.to_s
-        ret = bib_results_filter(result, year)
+        ret = bib_results_filter(result, code, year)
         if ret[:ret]
           item = ret[:ret].fetch
           warn "[relaton-ieee] (\"#{code}\") found #{item.docidentifier.first.id}"
@@ -42,9 +42,13 @@ module RelatonIeee
       # @param opts [Hash] options
       #
       # @return [Hash]
-      def bib_results_filter(result, year)
+      def bib_results_filter(result, ref, year)
+        rp1 = ref_parts ref
         missed_years = []
         result.each do |hit|
+          rp2 = ref_parts hit.hit["recordTitle"]
+          next if rp1[:code] != rp2[:code] || rp1[:corr] != rp2[:corr]
+
           return { ret: hit } if !year
 
           return { ret: hit } if year.to_i == hit.hit[:year]
@@ -54,6 +58,15 @@ module RelatonIeee
         { years: missed_years.uniq }
       end
 
+      def ref_parts(ref)
+        %r{
+          ^(?:IEEE\s(?:Std\s)?)?
+          (?<code>[^-/]+)
+          (?:-(?<year>\d{4}))?
+          (?:/(?<corr>\w+\s\d+-\d{4}))?
+        }x.match ref
+      end
+
       # @param code [Strig]
       # @param year [String]
       # @param missed_years [Array<Strig>]

data/lib/relaton_ieee/processor.rb

@@ -4,11 +4,12 @@ module RelatonIeee
   class Processor < Relaton::Processor
     attr_reader :idtype
 
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_ieee
       @prefix = "IEEE"
       @defaultprefix = %r{^IEEE\s}
       @idtype = "IEEE"
+      @datasets = %w[ieee-rawbib]
     end
 
     # @param code [String]
@@ -19,6 +20,18 @@ module RelatonIeee
       ::RelatonIeee::IeeeBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from ./iee-rawbib directory
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonIeee::IeeeBibliographicItem]
     def from_xml(xml)

data/lib/relaton_ieee/version.rb

@@ -1,3 +1,3 @@
 module RelatonIeee
-  VERSION = "1.7.4".freeze
+  VERSION = "1.9.1".freeze
 end

data/lib/relaton_ieee.rb

@@ -10,6 +10,7 @@ require "relaton_ieee/ieee_bibliographic_item"
 require "relaton_ieee/committee"
 require "relaton_ieee/xml_parser"
 require "relaton_ieee/hash_converter"
+require "relaton_ieee/data_fetcher"
 
 module RelatonIeee
   class Error < StandardError; end