relaton-itu 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fb722b2271d57d169b7f89c4ff87b0e16b3a9a25
-  data.tar.gz: 2755f77b6f20cb3f1f688ec1894132814f9401f0
+SHA256:
+  metadata.gz: 4ca4655d0b046848fd1ddd492e4e0d3e7ae53dd28bf491bc4cc049d2eccb5810
+  data.tar.gz: 40c7fbd808c557a460fbfed3811c4b87d51ed2dfb937a38a77b07c7c323ae554
 SHA512:
-  metadata.gz: 8df40643d1c03fbe71338a898fa19aa0b398ff32cfccd4c0065e864c5dc5e60fefb521fe16c319d1c173711a83850236d86e7ada4b1fbf654be32a994b598995
-  data.tar.gz: 8a41125691555e4b69775e302717f0b8f95fe675e465757b25bfc9f3ec7d7949fdd589e78252045658c21d74db812b883f25743d24ca64e911f1acd1c445ffda
+  metadata.gz: '08234d67ae2e8ec6e5461e80446e183c3482390a850b33e114fac0328b609409eb137eeb58cbf216e5025f4a4d97ea3fcad6d4e4cca8550400ac92ed916fee38'
+  data.tar.gz: 2c24f8c0b1fe4f4c25b94c128c51c939dc4980b9c5e38d0b48575c8705abb33d99b3b77ca1bc5222903048ab4b8f3ee6dfdd75c6fa0cdd6e25cf110ed7c4be0c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    relaton-itu (0.3.5)
+    relaton-itu (0.3.6)
       relaton-iso-bib (~> 0.3.0)
 
 GEM
@@ -35,10 +35,10 @@ GEM
       pry (~> 0.10)
     public_suffix (4.0.1)
     rake (10.5.0)
-    relaton-bib (0.3.9)
+    relaton-bib (0.3.11)
       addressable
       nokogiri
-    relaton-iso-bib (0.3.9)
+    relaton-iso-bib (0.3.11)
       isoics (~> 0.1.6)
       relaton-bib (~> 0.3.0)
       ruby_deep_clone (~> 0.8.0)

data/lib/relaton_itu/scrapper.rb

@@ -48,18 +48,18 @@ module RelatonItu
       # @return [Hash]
       # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       def parse_page(hit_data)
-        doc = get_page hit_data[:url]
+        url, doc = get_page hit_data[:url]
 
         # Fetch edition.
         edition = doc.at("//table/tr/td/span[contains(@id, 'Label8')]/b")&.text
 
         ItuBibliographicItem.new(
           fetched: Date.today.to_s,
-          docid: fetch_docid(hit_data[:code]),
+          docid: fetch_docid(doc),
           edition: edition,
           language: ["en"],
           script: ["Latn"],
-          title: fetch_titles(hit_data),
+          title: fetch_titles(doc),
           doctype: hit_data[:type],
           docstatus: fetch_status(doc),
           ics: [], # fetch_ics(doc),
@@ -68,7 +68,7 @@ module RelatonItu
           editorialgroup: fetch_workgroup(doc),
           abstract: fetch_abstract(doc),
           copyright: fetch_copyright(hit_data[:code], doc),
-          link: fetch_link(doc, hit_data[:url]),
+          link: fetch_link(doc, url),
           relation: fetch_relations(doc),
         )
       end
@@ -119,7 +119,7 @@ module RelatonItu
           uri = URI resp["location"]
           resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
         end
-        Nokogiri::HTML(resp.body)
+        [uri.to_s, Nokogiri::HTML(resp.body)]
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
              Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
              OpenSSL::SSL::SSLError
@@ -130,12 +130,15 @@ module RelatonItu
       # Fetch docid.
       # @param doc [Nokogiri::HTML::Document]
       # @return [Hash]
-      def fetch_docid(code)
-        # m = code.match(/(?<=\s)(?<project>[^\s]+)-?(?<part>(?<=-)\d+|)-?(?<subpart>(?<=-)\d+|)/)
-        # project_number: m[:project],
-        # part_number: m[:part],
-        # subpart_number: m[:subpart],
-        [RelatonBib::DocumentIdentifier.new(type: "ITU", id: code)]
+      def fetch_docid(doc)
+        doc.xpath(
+          "//span[@id='ctl00_content_main_uc_rec_main_info1_rpt_main_ctl00_lbl_rec']",
+          "//td[.='Identical standard:']/following-sibling::td",
+        ).map do |code|
+          id = code.text.match(%r{^.*?(?= \()}).to_s.squeeze(" ")
+          type = id.match(%r{^\w+}).to_s
+          RelatonBib::DocumentIdentifier.new(type: type, id: id)
+        end
       end
 
       # Fetch status.
@@ -186,16 +189,12 @@ module RelatonItu
       # @return [Array<Hash>]
       def fetch_relations(doc)
         doc.xpath('//div[contains(@id, "tab_sup")]//table/tr[position()>2]').map do |r|
-          r_type = r.at('./td/span[contains(@id, "Label4")]/nobr').text.downcase
-          type = case r_type
-                 when "in force" then "published"
-                 else r_type
-                 end
+          # r_type = r.at('./td/span[contains(@id, "Label4")]/nobr').text.downcase
           ref = r.at('./td/span[contains(@id, "title_e")]/nobr/a')
           # url = DOMAIN + ref[:href].sub(/^\./, "/ITU-T/recommendations")
           fref = RelatonBib::FormattedRef.new(content: ref.text, language: "en", script: "Latn")
           bibitem = RelatonIsoBib::IsoBibliographicItem.new(formattedref: fref)
-          { type: type, bibitem: bibitem }
+          { type: "complements", bibitem: bibitem }
         end
       end
       # rubocop:enable Metrics/MethodLength
@@ -208,10 +207,14 @@ module RelatonItu
       # end
 
       # Fetch titles.
-      # @param hit_data [Hash]
+      # @param doc [Nokogiri::HTML::Document]
       # @return [Array<Hash>]
-      def fetch_titles(hit_data)
-        titles = hit_data[:title].split " - "
+      def fetch_titles(doc)
+        # t = hit_data[:title].match(%r{(?<=\(\d{2}\/\d{4}\): ).*}).to_s
+        # t = hit_data[:title] if t.empty?
+        t = doc.at("//td[@class='title']")
+        return [] unless t
+        titles = t.text.split " - "
         case titles.size
         when 0
           intro, main, part = nil, "", nil
@@ -298,8 +301,8 @@ module RelatonItu
       # @return [Array<Hash>]
       def fetch_link(doc, url)
         links = [{ type: "src", content: url }]
-        obp_elms = doc.at('//table/tr/td/span[contains(@id, "Label4")]/a')
-        links << { type: "obp", content: DOMAIN + obp_elms[:href] } if obp_elms
+        obp_elms = doc.at('//a[@title="Persistent link to download the PDF file"]')
+        links << { type: "obp", content: DOMAIN + obp_elms[:href].strip } if obp_elms
         links
       end
 

data/lib/relaton_itu/version.rb

@@ -1,3 +1,3 @@
 module RelatonItu
-  VERSION = "0.3.5".freeze
+  VERSION = "0.3.6".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-itu
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-14 00:00:00.000000000 Z
+date: 2019-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -216,8 +216,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem