relaton-nist 0.5.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af67a70401309cab1e7d7b4511db20533a82927fc71aa4c439ee895bce140d2a
-  data.tar.gz: aa32fd8f52b1b3513153fdf6ee2d0898d16dfe4e86b2890303002aa0720274d3
+  metadata.gz: 06d8c511378ca4a23050ca5a6e1d65b8d78fda5f8104cb52da5619455e6dbfa2
+  data.tar.gz: 5587938b2ff4536214f5e159e6eecf04318c03aaaeaff0fd3cbe1dbd19740daf
 SHA512:
-  metadata.gz: d47764a549b16704b6672bed22add68438e0ef42be86f97a0d71dc2d3dbb8b371b96b1e120402ca2f9c3631ab57134dcf9341540b669bcdc7dcea11174a20336
-  data.tar.gz: 22e80353033d4b57d754a3b48b5515370cc1512a134bb534897c0e75cdcf72a3f1fe5444cc9e724aa6ac57bc6996ab825426547a513ebe4eb264cbecd03f354d
+  metadata.gz: e56fb78bdcee718e5c7543d6a0e6c5af6a45b739319f9ddaaf1dd233894b196b8bb4ea8d3442af2d4918f73e96d73ff78ebfbbc394da964e05078ab8375c012e
+  data.tar.gz: b673fbe4caa5e9bdd30868aac90d1fed5d146cbdc2ee6e2eb42da8b59ad53e38feaa66052b04fa0e2eb90b0889f866f3c61a635545168955c35e861d5bba9c91

data/.github/workflows/macos.yml

@@ -1,14 +1,19 @@
-# Auto-generated !!! Do not edit it manually
-# use ci-master https://github.com/metanorma/metanorma-build-scripts
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
 name: macos
 
-on: [push]
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ '**' ]
 
 jobs:
   test-macos:
     name: Test on Ruby ${{ matrix.ruby }} macOS
     runs-on: macos-latest
     strategy:
+      fail-fast: false
       matrix:
         ruby: [ '2.6', '2.5', '2.4' ]
     steps:
@@ -20,7 +25,7 @@ jobs:
           architecture: 'x64'
       - name: Update gems
         run: |
-          sudo gem install bundler -v "~> 2" --force
+          sudo gem install bundler --force
           bundle install --jobs 4 --retry 3
       - name: Run specs
         run: |

data/.github/workflows/ubuntu.yml

@@ -1,14 +1,19 @@
-# Auto-generated !!! Do not edit it manually
-# use ci-master https://github.com/metanorma/metanorma-build-scripts
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
 name: ubuntu
 
-on: [push]
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ '**' ]
 
 jobs:
   test-linux:
     name: Test on Ruby ${{ matrix.ruby }} Ubuntu
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         ruby: [ '2.6', '2.5', '2.4' ]
     steps:
@@ -20,7 +25,7 @@ jobs:
           architecture: 'x64'
       - name: Update gems
         run: |
-          gem install bundler -v "~> 2"
+          gem install bundler
           bundle install --jobs 4 --retry 3
       - name: Run specs
         run: |

data/.github/workflows/windows.yml

@@ -1,14 +1,19 @@
-# Auto-generated !!! Do not edit it manually
-# use ci-master https://github.com/metanorma/metanorma-build-scripts
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
 name: windows
 
-on: [push]
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ '**' ]
 
 jobs:
   test-windows:
     name: Test on Ruby ${{ matrix.ruby }} Windows
     runs-on: windows-latest
     strategy:
+      fail-fast: false
       matrix:
         ruby: [ '2.6', '2.5', '2.4' ]
     steps:
@@ -21,7 +26,7 @@ jobs:
       - name: Update gems
         shell: pwsh
         run: |
-          gem install bundler -v "~> 2"
+          gem install bundler
           bundle config --local path vendor/bundle
           bundle update
           bundle install --jobs 4 --retry 3

data/.gitignore

@@ -13,3 +13,4 @@
 .rspec_status
 pubs-export.json
 Gemfile.lock
+.byebug_history

data/lib/relaton_nist/hash_converter.rb

@@ -9,17 +9,12 @@ module RelatonNist
         ret = super
         return if ret.nil?
 
-        keyword_hash_to_bib(ret)
         commentperiod_hash_to_bib(ret)
         ret
       end
 
       private
 
-      def keyword_hash_to_bib(ret)
-        ret[:keyword]&.map! { |kw| Keyword.new kw }
-      end
-
       def commentperiod_hash_to_bib(ret)
         return unless ret[:commentperiod]
 

data/lib/relaton_nist/hit.rb

@@ -3,9 +3,6 @@
 module RelatonNist
   # Hit.
   class Hit < RelatonBib::Hit
-    # @return [RelatonNist::HitCollection]
-    attr_reader :hit_collection
-
     # Parse page.
     # @return [RelatonNist::NistBliographicItem]
     def fetch
@@ -16,17 +13,18 @@ module RelatonNist
     def sort_value
       @sort_value ||= begin
         sort_phrase = [hit[:serie], hit[:code], hit[:title]].join " "
-        corr = hit_collection.text.split.map do |w|
+        corr = hit_collection&.text&.split&.map do |w|
           if w =~ /\w+/ &&
               sort_phrase =~ Regexp.new(Regexp.escape(w), Regexp::IGNORECASE)
             1
           else 0
           end
-        end.sum
+        end&.sum.to_i
         corr + case hit[:status]
-               when "final" then 3
-               when "withdrawn" then 2
-               when "draft (withdrawn)" then 1
+               when "final" then 4
+               when "withdrawn" then 3
+               when "draft" then 2
+               when "draft (obsolete)" then 1
                else 0
                end
       end

data/lib/relaton_nist/hit_collection.rb

@@ -13,30 +13,24 @@ module RelatonNist
     DATAFILEDIR = File.expand_path ".relaton/nist", Dir.home
     DATAFILE = File.expand_path "pubs-export.zip", DATAFILEDIR
 
-    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-
     # @param ref_nbr [String]
     # @param year [String]
     # @param opts [Hash]
     # @option opts [String] :stage
     def initialize(ref_nbr, year = nil, opts = {})
-      @text = ref_nbr
-      @year = year
+      super ref_nbr, year
 
       /(?<docid>(SP|FIPS)\s[0-9-]+)/ =~ text
-      hits = docid ? from_json(docid, **opts) : from_csrc(**opts)
+      @array = docid ? from_json(docid, **opts) : from_csrc(**opts)
 
-      hits.sort! do |a, b|
+      @array.sort! do |a, b|
         if a.sort_value != b.sort_value
           b.sort_value - a.sort_value
         else
           (b.hit[:release_date] - a.hit[:release_date]).to_i
         end
       end
-      concat hits
-      @fetched = false
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     private
 
@@ -66,12 +60,12 @@ module RelatonNist
         serie = h.at("td[1]").text.strip
         code  = h.at("td[2]").text.strip
         title = link.text
-        url   = DOMAIN + link[:href]
+        doc_url   = DOMAIN + link[:href]
         status = h.at("td[4]").text.strip.downcase
         release_date = Date.strptime h.at("td[5]").text.strip, "%m/%d/%Y"
         Hit.new(
           {
-            code: code, serie: serie, title: title, url: url, status: status,
+            code: code, serie: serie, title: title, url: doc_url, status: status,
             release_date: release_date
           }, self
         )

data/lib/relaton_nist/nist_bibliographic_item.rb

@@ -4,7 +4,7 @@ module RelatonNist
     attr_reader :doctype
 
     # @return [Array<RelatonNist::Keyword>]
-    attr_reader :keyword
+    # attr_reader :keyword
 
     # @return [RelatonNist::CommentPeriod, NilClass]
     attr_reader :commentperiod
@@ -57,7 +57,7 @@ module RelatonNist
     # @option relation [Array<RelatonBib::BibItemLocality>] :bib_locality
     def initialize(**args)
       @doctype = args.delete(:doctype) || "standard"
-      @keyword = args.delete(:keyword) || []
+      # @keyword = args.delete(:keyword) || []
       @commentperiod = args.delete :commentperiod
       super
     end
@@ -78,7 +78,7 @@ module RelatonNist
     # @return [Hash]
     def to_hash
       hash = super
-      hash["keyword"] = single_element_array(keyword) if keyword&.any?
+      # hash["keyword"] = single_element_array(keyword) if keyword&.any?
       hash["commentperiod"] = commentperiod.to_hash if commentperiod
       hash
     end

data/lib/relaton_nist/nist_bibliography.rb

@@ -3,7 +3,6 @@ require "relaton_nist/nist_bibliographic_item"
 require "relaton_nist/scrapper"
 require "relaton_nist/hit_collection"
 require "relaton_nist/xml_parser"
-require "relaton_nist/keyword"
 require "relaton_nist/comment_period"
 require "relaton_nist/document_status"
 require "relaton_nist/hash_converter"

data/lib/relaton_nist/scrapper.rb

@@ -87,7 +87,12 @@ module RelatonNist
       # @return [Array<Nokogiri::HTML::Document, String>]
       def get_page(url)
         uri = URI url
-        resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
+        resp = Net::HTTP.get_response(uri)
+        %r{(?<=newLocation = 'https://' \+ window.location.hostname \+ ')(?<path>[^']+)} =~ resp.body
+        if path
+          uri = URI HitCollection::DOMAIN + path
+          resp = Net::HTTP.get_response(uri)
+        end
         Nokogiri::HTML(resp.body)
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
              Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
@@ -104,7 +109,7 @@ module RelatonNist
                    else
                      doc.at(
                        "//div[contains(@class, 'publications-detail')]/h3",
-                     )&.text&.strip
+                     )&.text&.strip.sub(/(?<=\w)\([^\)]+\)$/) { |m| " " + m.upcase}.squeeze " "
                    end
         item_ref ||= "?"
         [RelatonBib::DocumentIdentifier.new(id: item_ref, type: "NIST")]
@@ -129,7 +134,7 @@ module RelatonNist
           iter = doc["iteration"] == "initial" ? 1 : doc["iteration"]
         else
           case status
-          when "draft (withdrawn)"
+          when "draft (obsolete)"
             stage = "draft-public"
             subst = "withdrawn"
           when "retired draft"
@@ -138,7 +143,7 @@ module RelatonNist
           when "withdrawn"
             stage = "final"
             subst = "withdrawn"
-          when "draft"
+          when /^draft/
             stage = "draft-public"
             subst = "active"
           else
@@ -352,9 +357,9 @@ module RelatonNist
           links << { type: "uri", content: doc["uri"] } if doc["uri"]
           doi = "https://doi.org/" + doc["doi"] if doc["doi"]
         else
-          pub = doc.at "//p/strong[.='Publication:']"
-          pdf = pub.at "./following-sibling::a[.=' Local Download']"
-          doi = pub.at("./following-sibling::a[contains(.,'(DOI)')]")&.attr :href
+          pub = doc.at "//p/strong[contains(., 'Publication:')]"
+          pdf = pub&.at "./following-sibling::a[.=' Local Download']"
+          doi = pub&.at("./following-sibling::a[contains(.,'(DOI)')]")&.attr :href
           links << { type: "pdf", content: pdf[:href] } if pdf
         end
         links << { type: "doi", content: doi } if doi
@@ -417,10 +422,12 @@ module RelatonNist
                  else idx + 1
                  end
 
-          content = s.text.match(/^[^\(]+/).to_s.strip.gsub "  ", " "
+          content = s.text.match(/^[^\(]+/).to_s.strip.squeeze " "
 
-          ref = case content.match(/\w+/).to_s
-                when "Draft" then content.match(/(?<=Draft\s).+/).to_s + " (#{iter}PD)"
+          ref = case s.text
+                when /^Draft/ then content.match(/(?<=Draft\s).+/).to_s + " (#{iter}PD)"
+                when /\(Draft\)/ then content + " (#{iter}PD)"
+                else content
                 end
 
           fref = RelatonBib::FormattedRef.new(
@@ -438,7 +445,7 @@ module RelatonNist
               else
                 doc.xpath "//span[@id='pub-keywords-container']/span"
               end
-        kws.map { |kw| Keyword.new kw.is_a?(String) ? kw : kw.text }
+        kws.map { |kw| kw.is_a?(String) ? kw : kw.text }
       end
 
       # @param doc [Nokogiri::HTML::Document]

data/lib/relaton_nist/version.rb

@@ -1,3 +1,3 @@
 module RelatonNist
-  VERSION = "0.5.0".freeze
+  VERSION = "0.8.1".freeze
 end

data/lib/relaton_nist/xml_parser.rb

@@ -4,7 +4,11 @@ module RelatonNist
       def from_xml(xml)
         doc = Nokogiri::XML xml
         nistitem = doc.at("/bibitem|/bibdata")
-        NistBibliographicItem.new(item_data(nistitem))
+        if nistitem
+          NistBibliographicItem.new(item_data(nistitem))
+        elsif
+          warn "[relato-nist] can't find bibitem or bibdata element in the XML"
+        end
       end
 
       private
@@ -14,7 +18,6 @@ module RelatonNist
         ext = nistitem.at "./ext"
         return data unless ext
 
-        data[:keyword] = fetch_keyword(ext)
         data[:commentperiod] = fetch_commentperiod(ext)
         data
       end
@@ -39,12 +42,6 @@ module RelatonNist
           extended: cp.at("extended")&.text
         )
       end
-
-      def fetch_keyword(item)
-        item.xpath("./keyword").map do |kw|
-          Keyword.new kw.children.first.to_xml
-        end
-      end
     end
   end
 end

data/relaton_nist.gemspec

@@ -35,6 +35,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "ruby-jing"
 
-  spec.add_dependency "relaton-bib", "~> 0.5.0"
+  spec.add_dependency "relaton-bib", "~> 0.8.0"
   spec.add_dependency "rubyzip"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-nist
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-11 00:00:00.000000000 Z
+date: 2020-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -223,7 +223,6 @@ files:
 - lib/relaton_nist/hash_converter.rb
 - lib/relaton_nist/hit.rb
 - lib/relaton_nist/hit_collection.rb
-- lib/relaton_nist/keyword.rb
 - lib/relaton_nist/nist_bibliographic_item.rb
 - lib/relaton_nist/nist_bibliography.rb
 - lib/relaton_nist/processor.rb

data/lib/relaton_nist/keyword.rb

@@ -1,23 +0,0 @@
-module RelatonNist
-  class Keyword
-    include RelatonBib
-
-    # @return [Nokogiri::XML::DocumentFragment]
-    attr_reader :element
-
-    # @param element [String]
-    def initialize(element)
-      @element = Nokogiri::XML.fragment element
-    end
-
-    # @param builder [Nokogiri::XML::Builder]
-    def to_xml(builder)
-      builder.keyword element.to_xml
-    end
-
-    # @return [String]
-    def to_hash
-      element.text
-    end
-  end
-end