relaton-itu 1.7.7 → 1.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c9e267804ed407243d999f68830546b55db550bd511fa9146ab87e2db146c1d
-  data.tar.gz: c52daf864de7aeacd2651ca1a5442c0cadd8982e43ff671ae8a730a8a9559096
+  metadata.gz: 9478b4b00067cca1be679eea473b4db2b45f81129883cdacd0544ccfd3e7b680
+  data.tar.gz: a78f0bc199032f6762f659a534c605114f32de019b49332a2e9ecaf21f37fd7d
 SHA512:
-  metadata.gz: e412c86f5a31146d27ae6b05feef03c817e2004581efde5e3051a66b37fe061c1e984e417ba92e94ca15032bab17c3e6f70a2062842922c4b300dfc4224cac61
-  data.tar.gz: e6718282c40d4119d1b7f33bc0f553c611b87d8f46abe57ddf7eb8569f781d22fa54ea135808d7b895d2b9b48b267eafac4d8d25a43df841b35d21d5041dcaea
+  metadata.gz: 59581e2701ce5a5286165acbb2c263613bb3e88754bbab724204cb4e8c67c454bac6bda235bd1dc99582401d77366ac408d38e30f1af996f8877a67efbcf2ee8
+  data.tar.gz: 2771e587afe8badfda9da63199ef434760a76f76ad4088036a1e89237d6564547279321226efb79f69dc89a3576cb50055b3652db1dcde08eb7b89852d26ed8e

data/lib/relaton_itu.rb

@@ -1,3 +1,4 @@
+require "mechanize"
 require "relaton_itu/version"
 require "relaton_itu/itu_bibliography"
 require "digest/md5"

data/lib/relaton_itu/hash_converter.rb

@@ -3,6 +3,15 @@ module RelatonItu
     class << self
       private
 
+      #
+      # Ovverides superclass's method
+      #
+      # @param item [Hash]
+      # @retirn [RelatonItu::ItuBibliographicItem]
+      def bib_item(item)
+        ItuBibliographicItem.new(**item)
+      end
+
       def editorialgroup_hash_to_bib(ret)
         eg = ret[:editorialgroup]
         return unless eg

data/lib/relaton_itu/hit.rb

@@ -8,7 +8,7 @@ module RelatonItu
     # Parse page.
     # @return [RelatonItu::ItuBibliographicItem]
     def fetch
-      @fetch ||= Scrapper.parse_page hit, hit_collection.gi_imp
+      @fetch ||= Scrapper.parse_page self, hit_collection.gi_imp
     end
   end
 end

data/lib/relaton_itu/hit_collection.rb

@@ -12,50 +12,66 @@ module RelatonItu
     # @return [TrueClass, FalseClass]
     attr_reader :gi_imp
 
+    # @return [Mechanize]
+    attr_reader :agent
+
     # @param ref [String]
     # @param year [String]
-    def initialize(ref, year = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    def initialize(ref, year = nil) # rubocop:todo Metrics/MethodLength
       text = ref.sub /(?<=\.)Imp\s?(?=\d)/, ""
       super text, year
+      @agent = Mechanize.new
+      agent.user_agent_alias = "Mac Safari"
       @gi_imp = /\.Imp\d/.match?(ref)
-      if ref.match? /^(ITU-T|ITU-R\sRR)/
-        uri = URI "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
-        data = { json: params.to_json }
-        resp = Net::HTTP.post(uri, data.to_json,
-                              "Content-Type" => "application/json")
-        @array = hits JSON.parse(resp.body)
-      elsif ref.match? /^ITU-R/
-        rf = ref.sub(/^ITU-R\s/, "").upcase
-        url = "https://raw.githubusercontent.com/relaton/relaton-data-itu-r/master/data/#{rf}.yaml"
-        resp = Net::HTTP.get_response(URI(url))
-        if resp.code == "404"
-          @array = []
-          return
-        end
 
-        hash = YAML.safe_load resp.body
-        item_hash = HashConverter.hash_to_bib(hash)
-        item = ItuBibliographicItem.new **item_hash
-        hit = Hit.new({ url: url }, self)
-        hit.fetch = item
-        @array = [hit]
+      case ref
+      when /^(ITU-T|ITU-R\sRR)/
+        request_search
+      when /^ITU-R\s([-_.\w]+)$/
+        request_document($1.upcase)
       end
     end
 
     private
 
+    def request_search
+      url = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
+      data = { json: params.to_json }
+      resp = agent.post url, data.to_json, "Content-Type" => "application/json"
+      @array = hits JSON.parse(resp.body)
+    end
+
+    # @param ref [String] a document ref
+    def request_document(ref) # rubocop:todo Metrics/MethodLength
+      uri = URI::HTTPS.build(
+        host: "raw.githubusercontent.com",
+        path: "/relaton/relaton-data-itu-r/master/data/#{ref}.yaml"
+      )
+      resp = Net::HTTP.get_response(uri)
+      if resp.code == "404"
+        @array = []
+        return
+      end
+
+      hash = YAML.safe_load resp.body
+      item_hash = HashConverter.hash_to_bib(hash)
+      item = ItuBibliographicItem.new **item_hash
+      hit = Hit.new({ url: uri.to_s }, self)
+      hit.fetch = item
+      @array = [hit]
+    end
+
     # @return [String]
     def group
       @group ||= case text
-                 when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR} then "Publications"
+                 when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR}
+                   "Publications"
                  when %r{^ITU-T} then "Recommendations"
                  end
     end
 
-    # rubocop:disable Metrics/MethodLength
-
     # @return [Hash]
-    def params
+    def params # rubocop:disable Metrics/MethodLength
       {
         "Input" => text,
         "Start" => 0,
@@ -102,7 +118,6 @@ module RelatonItu
         "SearchType" => "All",
       }
     end
-    # rubocop:enable Metrics/MethodLength
 
     # @param data [Hash]
     # @return [Array<RelatonItu::Hit>]

data/lib/relaton_itu/itu_bibliography.rb

@@ -28,13 +28,14 @@ module RelatonItu
         HitCollection.new text, year
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
-             Net::ProtocolError, OpenSSL::SSL::SSLError
-        raise RelatonBib::RequestError, "Could not access http://www.itu.int"
+             Net::ProtocolError, URI::InvalidURIError => e
+        raise RelatonBib::RequestError, e.message
       end
 
       # @param code [String] the ISO standard Code to look up (e..g "ISO 9000")
       # @param year [String] the year the standard was published (optional)
-      # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
+      # @param opts [Hash] options; restricted to :all_parts if all-parts
+      #   reference is required
       # @return [String] Relaton XML serialisation of reference
       def get(code, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
         if year.nil?
@@ -61,12 +62,12 @@ module RelatonItu
         warn "[relaton-itu] WARNING: no match found online for #{id}. "\
           "The code must be exactly like it is on the standards website."
         unless missed_years.empty?
-          warn "[relaton-itu] (There was no match for #{year}, though there were matches "\
-            "found for #{missed_years.join(', ')}.)"
+          warn "[relaton-itu] (There was no match for #{year}, though there "\
+            "were matches found for #{missed_years.join(', ')}.)"
         end
         if /\d-\d/.match? code
-          warn "[relaton-itu] The provided document part may not exist, or the document "\
-            "may no longer be published in parts."
+          warn "[relaton-itu] The provided document part may not exist, or "\
+            "the document may no longer be published in parts."
         else
           warn "[relaton-itu] If you wanted to cite all document parts for the reference, "\
             "use \"#{code} (all parts)\".\nIf the document is not a standard, "\
@@ -123,7 +124,8 @@ module RelatonItu
       def isobib_results_filter(result, year)
         missed_years = []
         result.each do |r|
-          if !year || /\((\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
+          /\((\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
+          if !year || year == pyear
             ret = r.fetch
             return { ret: ret } if ret
           end

data/lib/relaton_itu/scrapper.rb

@@ -24,20 +24,18 @@ module RelatonItu
     }.freeze
 
     class << self
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-
       # Parse page.
-      # @param hit_data [Hash]
+      # @param hit [RelatonItu::Hit]
       # @return [Hash]
-      def parse_page(hit_data, imp = false)
-        url, doc = get_page hit_data[:url]
-        return unless doc
+      def parse_page(hit, imp = false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        doc = get_page hit
+        return unless doc.code == "200"
 
         if imp
           a = doc.at "//span[contains(@id, 'tab_ig_uc_rec')]/a"
           return unless a
 
-          url, doc = get_page URI.join(url, a[:href]).to_s
+          doc = get_page hit, a[:href].to_s
         end
 
         # Fetch edition.
@@ -46,36 +44,37 @@ module RelatonItu
         ItuBibliographicItem.new(
           fetched: Date.today.to_s,
           type: "standard",
-          docid: fetch_docid(doc, hit_data[:title]),
+          docid: fetch_docid(doc, hit.hit[:title]),
           edition: edition,
           language: ["en"],
           script: ["Latn"],
           title: fetch_titles(doc),
-          doctype: hit_data[:type],
+          doctype: hit.hit[:type],
           docstatus: fetch_status(doc),
           ics: [], # fetch_ics(doc),
           date: fetch_dates(doc),
-          contributor: fetch_contributors(hit_data[:code]),
-          editorialgroup: fetch_workgroup(hit_data[:code], doc),
-          abstract: fetch_abstract(doc),
-          copyright: fetch_copyright(hit_data[:code], doc),
-          link: fetch_link(doc, url),
+          contributor: fetch_contributors(hit.hit[:code]),
+          editorialgroup: fetch_workgroup(hit.hit[:code], doc),
+          abstract: fetch_abstract(doc, hit),
+          copyright: fetch_copyright(hit.hit[:code], doc),
+          link: fetch_link(doc),
           relation: fetch_relations(doc),
           place: ["Geneva"]
         )
       end
-      # rubocop:enable Metrics/AbcSize
 
       private
 
       # Fetch abstracts.
-      # @param doc [Nokigiri::HTML::Document]
-      # @return [Array<Array>]
-      def fetch_abstract(doc) # rubocop:disable Metrics/AbcSize
-        abstract_url = doc.at('//table/tr/td/span[contains(@id, "lbl_dms")]/div')
+      # @param doc [Mechanize::Page]
+      # @param hit [RelatonItu::Hit]
+      # @return [Array<Hash>]
+      def fetch_abstract(doc, hit) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        abstract_url = doc.at '//table/tr[td/span[.="In force"]]/td/span[contains(@id, "lbl_dms")]/div'
         content = if abstract_url
                     url = abstract_url[:onclick].match(/https?[^']+/).to_s
-                    d = Nokogiri::HTML Net::HTTP.get(URI(url)).encode(undef: :replace, replace: "")
+                    rsp = hit.hit_collection.agent.get url
+                    d = Nokogiri::HTML rsp.body.encode(undef: :replace, replace: "")
                     d.css("p.MsoNormal").text.gsub(/\r\n/, "").squeeze(" ").gsub(/\u00a0/, "")
                   elsif a = doc.at('//table/tr/td/span[contains(@class, "observation")]/text()')
                     a.text.strip
@@ -90,27 +89,20 @@ module RelatonItu
       end
 
       # Get page.
-      # @param path [String] page's path
+      # @param hit [RelatonItu::Hit]
+      # @param url [String, nil]
       # @return [Array<String, Nokogiri::HTML::Document>]
-      def get_page(url)
-        uri = URI url
-        resp = Net::HTTP.get_response(uri)
-        until resp.code == "200"
-          return if resp["location"] == "/en/publications/pages/notfound.aspx"
-
-          uri = URI resp["location"] if resp.code.match? /^30/
-          resp = Net::HTTP.get_response(uri)
-        end
-        [uri.to_s, Nokogiri::HTML(resp.body)]
+      def get_page(hit, url = nil)
+        uri = url || hit.hit[:url]
+        hit.hit_collection.agent.get uri
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
              Net::ProtocolError, OpenSSL::SSL::SSLError
-        raise RelatonBib::RequestError, "Could not access #{url}"
+        raise RelatonBib::RequestError, "Could not access #{uri}"
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Fetch docid.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @param title [String]
       # @return [Hash]
       def fetch_docid(doc, title)
@@ -123,6 +115,8 @@ module RelatonItu
         docids
       end
 
+      # @param text [String]
+      # @return [RelatonBib::DocumentIdentifier]
       def createdocid(text) # rubocop:disable Metrics/MethodLength
         %r{
           ^(?<code>((ITU-\w|ISO\/IEC)\s)?[^\(:]+)
@@ -140,7 +134,7 @@ module RelatonItu
       end
 
       # Fetch status.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [RelatonBib::DocumentStatus, NilClass]
       def fetch_status(doc)
         s = doc.at("//table/tr/td/span[contains(@id, 'Label7')]",
@@ -153,7 +147,7 @@ module RelatonItu
 
       # Fetch workgroup.
       # @param code [String]
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [RelatonItu::EditorialGroup, NilClass]
       def fetch_workgroup(code, doc)
         wg = doc.at('//table/tr/td/span[contains(@id, "Label8")]/a')
@@ -161,8 +155,7 @@ module RelatonItu
 
         group = wg && itugroup(wg.text)
         EditorialGroup.new(
-          bureau: code.match(/(?<=-)./).to_s,
-          group: group
+          bureau: code.match(/(?<=-)./).to_s, group: group
         )
       end
 
@@ -182,24 +175,24 @@ module RelatonItu
         ItuGroup.new name: name, type: type, acronym: acronym
       end
 
-      # rubocop:disable Metrics/MethodLength
-
       # Fetch relations.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_relations(doc)
-        doc.xpath('//div[contains(@id, "tab_sup")]//table/tr[position()>2]').map do |r|
+        doc.xpath('//div[contains(@id, "tab_sup")]//table/tr[position()>2]')
+          .map do |r|
           ref = r.at('./td/span[contains(@id, "title_e")]/nobr/a')
-          fref = RelatonBib::FormattedRef.new(content: ref.text, language: "en", script: "Latn")
-          bibitem = ItuBibliographicItem.new(formattedref: fref, type: "standard")
+          fref = RelatonBib::FormattedRef.new(content: ref.text, language: "en",
+                                              script: "Latn")
+          bibitem = ItuBibliographicItem.new(formattedref: fref,
+                                             type: "standard")
           { type: "complements", bibitem: bibitem }
         end
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Fetch titles.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<Hash>]
+      # @param doc [Mechanize::Page]
+      # @return [RelatonBib::TypedTitleStringCollection]
       def fetch_titles(doc)
         t = doc.at("//td[@class='title']|//div/table[1]/tr[4]/td/strong")
         return [] unless t
@@ -208,7 +201,7 @@ module RelatonItu
       end
 
       # Fetch dates
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_dates(doc) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
         dates = []
@@ -224,7 +217,7 @@ module RelatonItu
       end
 
       # Scrape Operational Bulletin date.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [String]
       def ob_date(doc)
         pdate = doc.at('//table/tbody/tr/td[contains(text(), "Year:")]')
@@ -246,7 +239,7 @@ module RelatonItu
       end
 
       # Fetch contributors
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_contributors(code)
         return [] unless code
@@ -262,11 +255,10 @@ module RelatonItu
       end
 
       # Fetch links.
-      # @param doc [Nokogiri::HTML::Document]
-      # @param url [String]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
-      def fetch_link(doc, url)
-        links = [{ type: "src", content: url }]
+      def fetch_link(doc)
+        links = [{ type: "src", content: doc.uri.to_s }]
         obp_elm = doc.at(
           '//a[@title="Persistent link to download the PDF file"]',
           "//font[contains(.,'PDF')]/../.."
@@ -277,6 +269,8 @@ module RelatonItu
         links
       end
 
+      # @param type [String]
+      # @param elm [Nokogiri::XML::Element]
       def typed_link(type, elm)
         {
           type: type,
@@ -286,7 +280,7 @@ module RelatonItu
 
       # Fetch copyright.
       # @param code [String]
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_copyright(code, doc)
         abbreviation = code.match(/^[^-]+/).to_s

data/lib/relaton_itu/version.rb

@@ -1,3 +1,3 @@
 module RelatonItu
-  VERSION = "1.7.7".freeze
+  VERSION = "1.8.1".freeze
 end

data/lib/relaton_itu/xml_parser.rb

@@ -33,7 +33,7 @@ module RelatonItu
         ItuGroup.new(
           type: group[:type],
           name: group.at("name").text,
-          acronym: group.at("acronym").text,
+          acronym: group.at("acronym")&.text,
           period: itugroupperiod(group.at("period")),
         )
       end

data/relaton-itu.gemspec

@@ -37,5 +37,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "vcr", "~> 5.0.0"
   spec.add_development_dependency "webmock"
 
-  spec.add_dependency "relaton-bib", "~> 1.7.0"
+  spec.add_dependency "mechanize", "~> 2.7.0"
+  spec.add_dependency "relaton-bib", "~> 1.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-itu
 version: !ruby/object:Gem::Version
-  version: 1.7.7
+  version: 1.8.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-04-14 00:00:00.000000000 Z
+date: 2021-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -122,20 +122,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.0
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: 1.8.0
 description: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem
   model'
 email:
@@ -193,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem