relaton-ieee 1.8.0 → 1.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f09ad9bd390643d4c3b77249151eda8d29c50a6a2af928cf075bccd91ac19b11
-  data.tar.gz: 12ea887aefc3f6e9c486391277deb04bb978171838be04a7ad3605c1774ed290
+  metadata.gz: f5ea9caf8eaf59f616cad6b137ff46fbfd0f28d38ad92c457467a3d19032d2a7
+  data.tar.gz: 88b376e90082b8a6bb730e5a07b96930b292c092ece0ce82850a50a9d9cdeb0e
 SHA512:
-  metadata.gz: 348bd013d0b3c7ca93cc1bf03d1b39c4814980832803fb97049d356dd7989e23407bb270e7d32e93d936e41930149ac55ad7e7d6d27cc4a7510c0ed563f2dbc0
-  data.tar.gz: d30bbe7e3c1ebb21414b7a148b3851892e62f64bcadbd3af0d0267ab898cb75dfa9af28def3295cb13740ddbd49e1e13435d4a455128c423c4d1b074ce828c1b
+  metadata.gz: c5eeb5339a3d598da5509de3ffc2bb9231694f9a9edc6603b40cb2634ded16f6ebee01c0ffb222b624426075c7dfd28382bdfac428077097821215a017680f7f
+  data.tar.gz: 9d8ee0f3c616d91d53ce9bfd54ceaf4a5cd458ac78471e69f7179daf6f5cf225681dadfe409011012f8c4e987e11f87e27a7e82b777aff2138873be5761aa617

data/.github/workflows/rake.yml

@@ -16,19 +16,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
+        ruby: [ '3.0', '2.7', '2.6', '2.5' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
-        include:
-          - ruby: '3.0'
-            os: 'ubuntu-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'windows-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'macos-latest'
-            experimental: true
     steps:
       - uses: actions/checkout@v2
         with:

data/.gitignore

@@ -11,4 +11,5 @@
 .rspec_status
 .rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
 .vscode/
+ieee-rawbib/
 Gemfile.lock

data/.rubocop.yml

@@ -5,6 +5,6 @@
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5
 Rails:
   Enabled: false

data/README.adoc

@@ -138,6 +138,16 @@ RelatonIeee::IeeeBibliography.get("IEEE 528-2019")
 ...
 ----
 
+=== Typed links
+
+Each IEEE document has `src` type link.
+
+[source,ruby]
+----
+item.link
+=> [#<RelatonBib::TypedUri:0x00007fe885219ba0 @content=#<Addressable::URI:0x8ac URI:https://standards.ieee.org/standard/528-2019.html>, @type="src">]
+----
+
 === Create bibliographic item from XML
 [source,ruby]
 ----
@@ -158,6 +168,25 @@ hash = YAML.load_file 'spec/fixtures/ieee_528_2019.yaml'
  ...
 ----
 
+=== Fetch data
+
+There is an IEEE dataset https://github.com/ietf-ribose/ieee-rawbib which can be converted into BibXML/BibYAML formats. The dataset needs to be placed into local directiory.
+
+The method `RelatonIeee::DataFetcher.fetch(output: "data", format: "yaml")` converts all the documents from the local `ieee-rawbib` directory and save them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonIeee::DataFetcher.fetch
+Started at: 2021-09-24 17:55:07 +0200
+Stopped at: 2021-09-24 17:57:30 +0200
+Done in: 143 sec.
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/grammars/biblio.rng

@@ -787,6 +787,7 @@
       <value>adapted</value>
       <value>vote-started</value>
       <value>vote-ended</value>
+      <value>announced</value>
     </choice>
   </define>
   <define name="bdate">

data/grammars/isodoc.rng

@@ -45,6 +45,11 @@
         <optional>
           <attribute name="alt"/>
         </optional>
+        <optional>
+          <attribute name="updatetype">
+            <data type="boolean"/>
+          </attribute>
+        </optional>
         <text/>
       </element>
     </define>
@@ -199,6 +204,18 @@
         </zeroOrMore>
       </element>
     </define>
+    <define name="dt">
+      <element name="dt">
+        <optional>
+          <attribute name="id">
+            <data type="ID"/>
+          </attribute>
+        </optional>
+        <zeroOrMore>
+          <ref name="TextElement"/>
+        </zeroOrMore>
+      </element>
+    </define>
     <define name="example">
       <element name="example">
         <attribute name="id">
@@ -543,6 +560,9 @@
     </define>
     <define name="BibDataExtensionType">
       <ref name="doctype"/>
+      <optional>
+        <ref name="docsubtype"/>
+      </optional>
       <optional>
         <ref name="editorialgroup"/>
       </optional>
@@ -890,6 +910,14 @@
     </define>
   </include>
   <!-- end overrides -->
+  <define name="docsubtype">
+    <element name="subdoctype">
+      <ref name="DocumentSubtype"/>
+    </element>
+  </define>
+  <define name="DocumentSubtype">
+    <text/>
+  </define>
   <define name="colgroup">
     <element name="colgroup">
       <oneOrMore>
@@ -939,7 +967,34 @@
   <define name="concept">
     <element name="concept">
       <optional>
-        <attribute name="term"/>
+        <attribute name="ital">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="ref">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
+      <optional>
+        <element name="refterm">
+          <zeroOrMore>
+            <choice>
+              <ref name="PureTextElement"/>
+              <ref name="stem"/>
+            </choice>
+          </zeroOrMore>
+        </element>
+      </optional>
+      <optional>
+        <element name="renderterm">
+          <zeroOrMore>
+            <choice>
+              <ref name="PureTextElement"/>
+              <ref name="stem"/>
+            </choice>
+          </zeroOrMore>
+        </element>
       </optional>
       <choice>
         <ref name="eref"/>
@@ -965,6 +1020,9 @@
       </attribute>
       <attribute name="name"/>
       <attribute name="action"/>
+      <optional>
+        <attribute name="class"/>
+      </optional>
       <zeroOrMore>
         <choice>
           <ref name="TextElement"/>
@@ -1191,13 +1249,17 @@
   </define>
   <define name="IsoWorkgroup">
     <optional>
-      <attribute name="number">
-        <data type="int"/>
-      </attribute>
+      <attribute name="number"/>
     </optional>
     <optional>
       <attribute name="type"/>
     </optional>
+    <optional>
+      <attribute name="identifier"/>
+    </optional>
+    <optional>
+      <attribute name="prefix"/>
+    </optional>
     <text/>
   </define>
   <define name="ics">
@@ -1459,26 +1521,26 @@
     <optional>
       <ref name="section-title"/>
     </optional>
-    <group>
+    <choice>
       <choice>
         <group>
-          <zeroOrMore>
+          <oneOrMore>
             <ref name="BasicBlock"/>
-          </zeroOrMore>
+          </oneOrMore>
           <zeroOrMore>
             <ref name="note"/>
           </zeroOrMore>
         </group>
         <ref name="amend"/>
       </choice>
-      <zeroOrMore>
+      <oneOrMore>
         <choice>
           <ref name="clause-subsection"/>
           <ref name="terms"/>
           <ref name="definitions"/>
         </choice>
-      </zeroOrMore>
-    </group>
+      </oneOrMore>
+    </choice>
   </define>
   <define name="Annex-Section">
     <optional>

data/lib/relaton_ieee/bibxml_parser.rb

@@ -0,0 +1,14 @@
+module RelatonIeee
+  module BibXMLParser
+    extend RelatonBib::BibXMLParser
+    extend BibXMLParser
+
+    FLAVOR = "IEEE".freeze
+
+    # @param attrs [Hash]
+    # @return [RelatonBib::IetfBibliographicItem]
+    def bib_item(**attrs)
+      IeeeBibliographicItem.new(**attrs)
+    end
+  end
+end

data/lib/relaton_ieee/data_fetcher.rb

@@ -0,0 +1,229 @@
+require "zip"
+require "relaton_ieee/data_parser"
+require "relaton_ieee/rawbib_id_parser"
+
+module RelatonIeee
+  class DataFetcher
+    RELATION_TYPES = {
+      "S" => { type: "obsoletedBy" },
+      "V" => { type: "updates", description: "revises" },
+      "T" => { type: "updates", description: "amends" },
+      "C" => { type: "updates", description: "corrects" },
+      "O" => { type: "adoptedFrom" },
+      "P" => { type: "complementOf", description: "supplement" },
+      "N" => false, "G" => false,
+      "F" => false, "I" => false,
+      "E" => false, "B" => false, "W" => false
+    }.freeze
+
+    # @return [Hash] list of AMSID => PubID
+    attr_reader :backrefs
+
+    #
+    # Create RelatonIeee::DataFetcher instance
+    #
+    # @param [String] output output dir
+    # @param [Strong] format output format. Allowed values: "yaml" or "xml"
+    #
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+      @crossrefs = {}
+      @backrefs = {}
+      # @normtitles = []
+    end
+
+    #
+    # Convert documents from `ieee-rawbib` dir (IEEE dataset) to BibYAML/BibXML
+    #
+    # @param [String] output ('data') output dir
+    # @param [String] format ('yaml') output format.
+    #   Allowed values: "yaml" or "xml"
+    #
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # Convert documents from `ieee-rawbib` dir (IEEE dataset) to BibYAML/BibXML
+    #
+    def fetch # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      Dir["ieee-rawbib/**/*.{xml,zip}"].reject { |f| f["Deleted_"] }.each do |f|
+        xml = case File.extname(f)
+              when ".zip" then read_zip f
+              when ".xml" then File.read f, encoding: "UTF-8"
+              end
+        fetch_doc xml, f
+      rescue StandardError => e
+        warn "File: #{f}"
+        warn e.message
+        warn e.backtrace
+      end
+      # File.write "normtitles.txt", @normtitles.join("\n")
+      update_relations
+    end
+
+    #
+    # Extract XML file from zip archive
+    #
+    # @param [String] file path to achive
+    #
+    # @return [String] file content
+    #
+    def read_zip(file)
+      Zip::File.open(file) do |zf|
+        entry = zf.glob("**/*.xml").first
+        entry.get_input_stream.read
+      end
+    end
+
+    #
+    # Parse document and save it
+    #
+    # @param [String] xml content
+    # @param [String] filename source file
+    #
+    def fetch_doc(xml, filename) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      doc = Nokogiri::XML(xml).at("/publication")
+      unless doc
+        warn "Empty file: #{filename}"
+        return
+      end
+      stdid = doc.at("./publicationinfo/standard_id").text
+      if stdid == "0"
+        # nt = doc&.at("./normtitle")&.text
+        # ntid = @normtitles.index nt
+        # @normtitles << nt if nt && !ntid
+        warn "Zero standard_id in #{filename}"
+        return
+      end
+      bib = DataParser.parse doc, self
+      if bib.docnumber.nil?
+        nt = doc&.at("./normtitle")&.text
+        warn "PubID parse error. Normtitle: #{nt}, file: #{filename}"
+        return
+      end
+      amsid = doc.at("./publicationinfo/amsid").text
+      if backrefs.value?(bib.docidentifier[0].id) && /updates\.\d+/ !~ filename
+        oamsid = backrefs.key bib.docidentifier[0].id
+        warn "Document exists ID: \"#{bib.docidentifier[0].id}\" AMSID: "\
+             "\"#{amsid}\" source: \"#{filename}\". Other AMSID: \"#{oamsid}\""
+        if bib.docidentifier[0].id.include?(doc.at("./publicationinfo/stdnumber").text)
+          save_doc bib # rewrite file if the PubID matches to the stdnumber
+          backrefs[amsid] = bib.docidentifier[0].id
+        end
+      else
+        save_doc bib
+        backrefs[amsid] = bib.docidentifier[0].id
+      end
+    end
+
+    #
+    # Save unresolved relation reference
+    #
+    # @param [String] docnumber of main document
+    # @param [Nokogiri::XML::Element] amsid relation data
+    #
+    def add_crossref(docnumber, amsid)
+      return if RELATION_TYPES[amsid[:type]] == false
+
+      ref = { amsid: amsid.text, type: amsid[:type] }
+      if @crossrefs[docnumber]
+        @crossrefs[docnumber] << ref
+      else @crossrefs[docnumber] = [ref]
+      end
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [RelatonIeee::IeeeBibliographicItem] bib
+    #
+    def save_doc(bib)
+      c = case @format
+          when "xml" then bib.to_xml(bibdata: true)
+          when "yaml" then bib.to_hash.to_yaml
+          else bib.send("to_#{@format}")
+          end
+      File.write file_name(bib.docnumber), c, encoding: "UTF-8"
+    end
+
+    #
+    # Make filename from PubID
+    #
+    # @param [String] docnumber
+    #
+    # @return [String] filename
+    #
+    def file_name(docnumber)
+      name = docnumber.gsub(/\s-/, "-").gsub(/[\s,:\/]/, "_").squeeze("_").upcase
+      File.join @output, "#{name}.#{@ext}"
+    end
+
+    #
+    # Update unresoverd relations
+    #
+    def update_relations # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      @crossrefs.each do |dnum, rfs|
+        bib = nil
+        rfs.each do |rf|
+          if backrefs[rf[:amsid]]
+            rel = create_relation(rf[:type], backrefs[rf[:amsid]])
+            if rel
+              bib ||= read_bib(dnum)
+              bib.relation << rel
+              save_doc bib
+            end
+          else
+            warn "Unresolved relation: '#{rf[:amsid]}' type: '#{rf[:type]}' for '#{dnum}'"
+          end
+        end
+      end
+    end
+
+    #
+    # Create relation instance
+    #
+    # @param [String] type IEEE relation type
+    # @param [String] fref reference
+    #
+    # @return [RelatonBib::DocumentRelation]
+    #
+    def create_relation(type, fref)
+      return if RELATION_TYPES[type] == false
+
+      fr = RelatonBib::FormattedRef.new(content: fref)
+      bib = IeeeBibliographicItem.new formattedref: fr
+      desc = RELATION_TYPES[type][:description]
+      description = desc && RelatonBib::FormattedString.new(content: desc, language: "en", script: "Latn")
+      RelatonBib::DocumentRelation.new(
+        type: RELATION_TYPES[type][:type],
+        description: description,
+        bibitem: bib,
+      )
+    end
+
+    #
+    # Read document form BibXML/BibYAML file
+    #
+    # @param [String] docnumber
+    #
+    # @return [RelatonIeee::IeeeBibliographicItem]
+    #
+    def read_bib(docnumber)
+      c = File.read file_name(docnumber), encoding: "UTF-8"
+      case @format
+      when "xml" then XMLParser.from_xml c
+      when "bibxml" then BibXMLParser.parse c
+      else IeeeBibliographicItem.from_hash YAML.safe_load(c)
+      end
+    end
+  end
+end