relaton-itu 1.7.7 → 1.7.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c9e267804ed407243d999f68830546b55db550bd511fa9146ab87e2db146c1d
-  data.tar.gz: c52daf864de7aeacd2651ca1a5442c0cadd8982e43ff671ae8a730a8a9559096
+  metadata.gz: 249fe896ec8a77979ca15d6a42da98ad2ac2620cfe8dc0f468cd14277c5a35b0
+  data.tar.gz: 62415ed835abc49cf00d3048b52556b3f718a4ad0dc531ec2c20572b95305210
 SHA512:
-  metadata.gz: e412c86f5a31146d27ae6b05feef03c817e2004581efde5e3051a66b37fe061c1e984e417ba92e94ca15032bab17c3e6f70a2062842922c4b300dfc4224cac61
-  data.tar.gz: e6718282c40d4119d1b7f33bc0f553c611b87d8f46abe57ddf7eb8569f781d22fa54ea135808d7b895d2b9b48b267eafac4d8d25a43df841b35d21d5041dcaea
+  metadata.gz: 65a5bcf91f851cc4ec3139fad83b0c83f143b1bafefe8c9638e34072f7d82f76b77517fedf153fcdf7b63ee903fcd372c64e378edaeb231835f477f004f7e94a
+  data.tar.gz: 61cdc7df34b24f5d3f3e56b967e9e2b34337bc164b32691747853660cbb6c9e337a6eee058162f97e218da356f4028d7c107b4be4ed7dc10f99253e6985fccd9

data/lib/relaton_itu.rb

@@ -1,3 +1,4 @@
+require "mechanize"
 require "relaton_itu/version"
 require "relaton_itu/itu_bibliography"
 require "digest/md5"

data/lib/relaton_itu/hash_converter.rb

@@ -3,6 +3,15 @@ module RelatonItu
     class << self
       private
 
+      #
+      # Ovverides superclass's method
+      #
+      # @param item [Hash]
+      # @retirn [RelatonItu::ItuBibliographicItem]
+      def bib_item(item)
+        ItuBibliographicItem.new(**item)
+      end
+
       def editorialgroup_hash_to_bib(ret)
         eg = ret[:editorialgroup]
         return unless eg

data/lib/relaton_itu/hit.rb

@@ -8,7 +8,7 @@ module RelatonItu
     # Parse page.
     # @return [RelatonItu::ItuBibliographicItem]
     def fetch
-      @fetch ||= Scrapper.parse_page hit, hit_collection.gi_imp
+      @fetch ||= Scrapper.parse_page self, hit_collection.gi_imp
     end
   end
 end

data/lib/relaton_itu/hit_collection.rb

@@ -12,17 +12,21 @@ module RelatonItu
     # @return [TrueClass, FalseClass]
     attr_reader :gi_imp
 
+    # @return [Mechanize]
+    attr_reader :agent
+
     # @param ref [String]
     # @param year [String]
     def initialize(ref, year = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       text = ref.sub /(?<=\.)Imp\s?(?=\d)/, ""
       super text, year
+      @agent = Mechanize.new
+      agent.user_agent_alias = "Mac Safari"
       @gi_imp = /\.Imp\d/.match?(ref)
       if ref.match? /^(ITU-T|ITU-R\sRR)/
-        uri = URI "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
+        url = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
         data = { json: params.to_json }
-        resp = Net::HTTP.post(uri, data.to_json,
-                              "Content-Type" => "application/json")
+        resp = agent.post url, data.to_json, "Content-Type" => "application/json"
         @array = hits JSON.parse(resp.body)
       elsif ref.match? /^ITU-R/
         rf = ref.sub(/^ITU-R\s/, "").upcase
@@ -47,7 +51,8 @@ module RelatonItu
     # @return [String]
     def group
       @group ||= case text
-                 when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR} then "Publications"
+                 when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR}
+                   "Publications"
                  when %r{^ITU-T} then "Recommendations"
                  end
     end

data/lib/relaton_itu/itu_bibliography.rb

@@ -28,13 +28,14 @@ module RelatonItu
         HitCollection.new text, year
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
-             Net::ProtocolError, OpenSSL::SSL::SSLError
-        raise RelatonBib::RequestError, "Could not access http://www.itu.int"
+             Net::ProtocolError, URI::InvalidURIError => e
+        raise RelatonBib::RequestError, e.message
       end
 
       # @param code [String] the ISO standard Code to look up (e..g "ISO 9000")
       # @param year [String] the year the standard was published (optional)
-      # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
+      # @param opts [Hash] options; restricted to :all_parts if all-parts
+      #   reference is required
       # @return [String] Relaton XML serialisation of reference
       def get(code, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
         if year.nil?
@@ -61,12 +62,12 @@ module RelatonItu
         warn "[relaton-itu] WARNING: no match found online for #{id}. "\
           "The code must be exactly like it is on the standards website."
         unless missed_years.empty?
-          warn "[relaton-itu] (There was no match for #{year}, though there were matches "\
-            "found for #{missed_years.join(', ')}.)"
+          warn "[relaton-itu] (There was no match for #{year}, though there "\
+            "were matches found for #{missed_years.join(', ')}.)"
         end
         if /\d-\d/.match? code
-          warn "[relaton-itu] The provided document part may not exist, or the document "\
-            "may no longer be published in parts."
+          warn "[relaton-itu] The provided document part may not exist, or "\
+            "the document may no longer be published in parts."
         else
           warn "[relaton-itu] If you wanted to cite all document parts for the reference, "\
             "use \"#{code} (all parts)\".\nIf the document is not a standard, "\
@@ -123,7 +124,8 @@ module RelatonItu
       def isobib_results_filter(result, year)
         missed_years = []
         result.each do |r|
-          if !year || /\((\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
+          /\((\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
+          if !year || year == pyear
             ret = r.fetch
             return { ret: ret } if ret
           end

data/lib/relaton_itu/scrapper.rb

@@ -24,20 +24,18 @@ module RelatonItu
     }.freeze
 
     class << self
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-
       # Parse page.
-      # @param hit_data [Hash]
+      # @param hit [RelatonItu::Hit]
       # @return [Hash]
-      def parse_page(hit_data, imp = false)
-        url, doc = get_page hit_data[:url]
-        return unless doc
+      def parse_page(hit, imp = false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        doc = get_page hit
+        return unless doc.code == "200"
 
         if imp
           a = doc.at "//span[contains(@id, 'tab_ig_uc_rec')]/a"
           return unless a
 
-          url, doc = get_page URI.join(url, a[:href]).to_s
+          doc = get_page hit, a[:href].to_s
         end
 
         # Fetch edition.
@@ -46,36 +44,37 @@ module RelatonItu
         ItuBibliographicItem.new(
           fetched: Date.today.to_s,
           type: "standard",
-          docid: fetch_docid(doc, hit_data[:title]),
+          docid: fetch_docid(doc, hit.hit[:title]),
           edition: edition,
           language: ["en"],
           script: ["Latn"],
           title: fetch_titles(doc),
-          doctype: hit_data[:type],
+          doctype: hit.hit[:type],
           docstatus: fetch_status(doc),
           ics: [], # fetch_ics(doc),
           date: fetch_dates(doc),
-          contributor: fetch_contributors(hit_data[:code]),
-          editorialgroup: fetch_workgroup(hit_data[:code], doc),
-          abstract: fetch_abstract(doc),
-          copyright: fetch_copyright(hit_data[:code], doc),
-          link: fetch_link(doc, url),
+          contributor: fetch_contributors(hit.hit[:code]),
+          editorialgroup: fetch_workgroup(hit.hit[:code], doc),
+          abstract: fetch_abstract(doc, hit),
+          copyright: fetch_copyright(hit.hit[:code], doc),
+          link: fetch_link(doc),
           relation: fetch_relations(doc),
           place: ["Geneva"]
         )
       end
-      # rubocop:enable Metrics/AbcSize
 
       private
 
       # Fetch abstracts.
-      # @param doc [Nokigiri::HTML::Document]
-      # @return [Array<Array>]
-      def fetch_abstract(doc) # rubocop:disable Metrics/AbcSize
-        abstract_url = doc.at('//table/tr/td/span[contains(@id, "lbl_dms")]/div')
+      # @param doc [Mechanize::Page]
+      # @param hit [RelatonItu::Hit]
+      # @return [Array<Hash>]
+      def fetch_abstract(doc, hit) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        abstract_url = doc.at '//table/tr/td/span[contains(@id, "lbl_dms")]/div'
         content = if abstract_url
                     url = abstract_url[:onclick].match(/https?[^']+/).to_s
-                    d = Nokogiri::HTML Net::HTTP.get(URI(url)).encode(undef: :replace, replace: "")
+                    rsp = hit.hit_collection.agent.get url
+                    d = Nokogiri::HTML rsp.body.encode(undef: :replace, replace: "")
                     d.css("p.MsoNormal").text.gsub(/\r\n/, "").squeeze(" ").gsub(/\u00a0/, "")
                   elsif a = doc.at('//table/tr/td/span[contains(@class, "observation")]/text()')
                     a.text.strip
@@ -90,27 +89,20 @@ module RelatonItu
       end
 
       # Get page.
-      # @param path [String] page's path
+      # @param hit [RelatonItu::Hit]
+      # @param url [String, nil]
       # @return [Array<String, Nokogiri::HTML::Document>]
-      def get_page(url)
-        uri = URI url
-        resp = Net::HTTP.get_response(uri)
-        until resp.code == "200"
-          return if resp["location"] == "/en/publications/pages/notfound.aspx"
-
-          uri = URI resp["location"] if resp.code.match? /^30/
-          resp = Net::HTTP.get_response(uri)
-        end
-        [uri.to_s, Nokogiri::HTML(resp.body)]
+      def get_page(hit, url = nil)
+        uri = url || hit.hit[:url]
+        hit.hit_collection.agent.get uri
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
              Net::ProtocolError, OpenSSL::SSL::SSLError
-        raise RelatonBib::RequestError, "Could not access #{url}"
+        raise RelatonBib::RequestError, "Could not access #{uri}"
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Fetch docid.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @param title [String]
       # @return [Hash]
       def fetch_docid(doc, title)
@@ -123,6 +115,8 @@ module RelatonItu
         docids
       end
 
+      # @param text [String]
+      # @return [RelatonBib::DocumentIdentifier]
       def createdocid(text) # rubocop:disable Metrics/MethodLength
         %r{
           ^(?<code>((ITU-\w|ISO\/IEC)\s)?[^\(:]+)
@@ -140,7 +134,7 @@ module RelatonItu
       end
 
       # Fetch status.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [RelatonBib::DocumentStatus, NilClass]
       def fetch_status(doc)
         s = doc.at("//table/tr/td/span[contains(@id, 'Label7')]",
@@ -153,7 +147,7 @@ module RelatonItu
 
       # Fetch workgroup.
       # @param code [String]
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [RelatonItu::EditorialGroup, NilClass]
       def fetch_workgroup(code, doc)
         wg = doc.at('//table/tr/td/span[contains(@id, "Label8")]/a')
@@ -161,8 +155,7 @@ module RelatonItu
 
         group = wg && itugroup(wg.text)
         EditorialGroup.new(
-          bureau: code.match(/(?<=-)./).to_s,
-          group: group
+          bureau: code.match(/(?<=-)./).to_s, group: group
         )
       end
 
@@ -182,24 +175,24 @@ module RelatonItu
         ItuGroup.new name: name, type: type, acronym: acronym
       end
 
-      # rubocop:disable Metrics/MethodLength
-
       # Fetch relations.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_relations(doc)
-        doc.xpath('//div[contains(@id, "tab_sup")]//table/tr[position()>2]').map do |r|
+        doc.xpath('//div[contains(@id, "tab_sup")]//table/tr[position()>2]')
+          .map do |r|
           ref = r.at('./td/span[contains(@id, "title_e")]/nobr/a')
-          fref = RelatonBib::FormattedRef.new(content: ref.text, language: "en", script: "Latn")
-          bibitem = ItuBibliographicItem.new(formattedref: fref, type: "standard")
+          fref = RelatonBib::FormattedRef.new(content: ref.text, language: "en",
+                                              script: "Latn")
+          bibitem = ItuBibliographicItem.new(formattedref: fref,
+                                             type: "standard")
           { type: "complements", bibitem: bibitem }
         end
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Fetch titles.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<Hash>]
+      # @param doc [Mechanize::Page]
+      # @return [RelatonBib::TypedTitleStringCollection]
       def fetch_titles(doc)
         t = doc.at("//td[@class='title']|//div/table[1]/tr[4]/td/strong")
         return [] unless t
@@ -208,7 +201,7 @@ module RelatonItu
       end
 
       # Fetch dates
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_dates(doc) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
         dates = []
@@ -224,7 +217,7 @@ module RelatonItu
       end
 
       # Scrape Operational Bulletin date.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [String]
       def ob_date(doc)
         pdate = doc.at('//table/tbody/tr/td[contains(text(), "Year:")]')
@@ -246,7 +239,7 @@ module RelatonItu
       end
 
       # Fetch contributors
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_contributors(code)
         return [] unless code
@@ -262,11 +255,10 @@ module RelatonItu
       end
 
       # Fetch links.
-      # @param doc [Nokogiri::HTML::Document]
-      # @param url [String]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
-      def fetch_link(doc, url)
-        links = [{ type: "src", content: url }]
+      def fetch_link(doc)
+        links = [{ type: "src", content: doc.uri.to_s }]
         obp_elm = doc.at(
           '//a[@title="Persistent link to download the PDF file"]',
           "//font[contains(.,'PDF')]/../.."
@@ -277,6 +269,8 @@ module RelatonItu
         links
       end
 
+      # @param type [String]
+      # @param elm [Nokogiri::XML::Element]
       def typed_link(type, elm)
         {
           type: type,
@@ -286,7 +280,7 @@ module RelatonItu
 
       # Fetch copyright.
       # @param code [String]
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_copyright(code, doc)
         abbreviation = code.match(/^[^-]+/).to_s

data/lib/relaton_itu/version.rb

@@ -1,3 +1,3 @@
 module RelatonItu
-  VERSION = "1.7.7".freeze
+  VERSION = "1.7.8".freeze
 end

data/relaton-itu.gemspec

@@ -37,5 +37,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "vcr", "~> 5.0.0"
   spec.add_development_dependency "webmock"
 
+  spec.add_dependency "mechanize"
   spec.add_dependency "relaton-bib", "~> 1.7.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-itu
 version: !ruby/object:Gem::Version
-  version: 1.7.7
+  version: 1.7.8
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-04-14 00:00:00.000000000 Z
+date: 2021-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement