relaton-itu 1.7.6 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b5d9ccc48e23558a34cc5af40c8603e64cf7a4ea5d51d7996ea17a4996cfaeec
-  data.tar.gz: fa28142b10ee1ef854584ee52854642305c86ad78ff32dc059f30315783ccd9b
+  metadata.gz: 2d8d0bba9feeb4b7bc45281e572f44b5db041807f22693fdd64a14427037af52
+  data.tar.gz: e57274331dd84d03f72b27bac3c8f916e0e74bf40487d15d5b98197d9fa0c96a
 SHA512:
-  metadata.gz: b304791f0d90fa0f23c378d30988f7ef1fa4d2abd36c2487fae454bba257019c0811f31354b94d8fb756dfb26eced93ac251d2ba5e64a41edfbc4268129edb08
-  data.tar.gz: 4535c1832d38d37f7df709abac6cc52e446f1da1be1704a8a3e4eaa0a98b47667976f88410bf54b73e6e10f50fa5f0807891f23775af772e6ff6d42ee54db18b
+  metadata.gz: 474d54e7621972e39a45b1924a8c10c1c65bc2b98d353c922033d32ff7584bea6bba69e7a69b3e453e80d176be0a3fd6a33924e7572eb50ccde51cb63d40b03c
+  data.tar.gz: 606f4346ef69872ba6a03c6ebddee02b0dc644b5cf2a0153929d0175ba4898d4f9b6a68cf6fb1bc3dc719486bbebd37596de512cf5657a8962cd376fd8ac60c3

data/lib/relaton_itu.rb

@@ -1,3 +1,4 @@
+require "mechanize"
 require "relaton_itu/version"
 require "relaton_itu/itu_bibliography"
 require "digest/md5"

data/lib/relaton_itu/hash_converter.rb

@@ -3,6 +3,15 @@ module RelatonItu
     class << self
       private
 
+      #
+      # Ovverides superclass's method
+      #
+      # @param item [Hash]
+      # @retirn [RelatonItu::ItuBibliographicItem]
+      def bib_item(item)
+        ItuBibliographicItem.new(**item)
+      end
+
       def editorialgroup_hash_to_bib(ret)
         eg = ret[:editorialgroup]
         return unless eg

data/lib/relaton_itu/hit.rb

@@ -8,7 +8,7 @@ module RelatonItu
     # Parse page.
     # @return [RelatonItu::ItuBibliographicItem]
     def fetch
-      @fetch ||= Scrapper.parse_page hit, hit_collection.gi_imp
+      @fetch ||= Scrapper.parse_page self, hit_collection.gi_imp
     end
   end
 end

data/lib/relaton_itu/hit_collection.rb

@@ -12,50 +12,66 @@ module RelatonItu
     # @return [TrueClass, FalseClass]
     attr_reader :gi_imp
 
+    # @return [Mechanize]
+    attr_reader :agent
+
     # @param ref [String]
     # @param year [String]
-    def initialize(ref, year = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    def initialize(ref, year = nil) # rubocop:todo Metrics/MethodLength
       text = ref.sub /(?<=\.)Imp\s?(?=\d)/, ""
       super text, year
+      @agent = Mechanize.new
+      agent.user_agent_alias = "Mac Safari"
       @gi_imp = /\.Imp\d/.match?(ref)
-      if ref.match? /^(ITU-T|ITU-R\sRR)/
-        uri = URI "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
-        data = { json: params.to_json }
-        resp = Net::HTTP.post(uri, data.to_json,
-                              "Content-Type" => "application/json")
-        @array = hits JSON.parse(resp.body)
-      elsif ref.match? /^ITU-R/
-        rf = ref.sub(/^ITU-R\s/, "").upcase
-        url = "https://raw.githubusercontent.com/relaton/relaton-data-itu-r/master/data/#{rf}.yaml"
-        resp = Net::HTTP.get_response(URI(url))
-        if resp.code == "404"
-          @array = []
-          return
-        end
 
-        hash = YAML.safe_load resp.body
-        item_hash = HashConverter.hash_to_bib(hash)
-        item = ItuBibliographicItem.new **item_hash
-        hit = Hit.new({ url: url }, self)
-        hit.fetch = item
-        @array = [hit]
+      case ref
+      when /^(ITU-T|ITU-R\sRR)/
+        request_search
+      when /^ITU-R\s([-_.\w]+)$/
+        request_document($1.upcase)
       end
     end
 
     private
 
+    def request_search
+      url = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
+      data = { json: params.to_json }
+      resp = agent.post url, data.to_json, "Content-Type" => "application/json"
+      @array = hits JSON.parse(resp.body)
+    end
+
+    # @param ref [String] a document ref
+    def request_document(ref) # rubocop:todo Metrics/MethodLength
+      uri = URI::HTTPS.build(
+        host: "raw.githubusercontent.com",
+        path: "/relaton/relaton-data-itu-r/master/data/#{ref}.yaml"
+      )
+      resp = Net::HTTP.get_response(uri)
+      if resp.code == "404"
+        @array = []
+        return
+      end
+
+      hash = YAML.safe_load resp.body
+      item_hash = HashConverter.hash_to_bib(hash)
+      item = ItuBibliographicItem.new **item_hash
+      hit = Hit.new({ url: uri.to_s }, self)
+      hit.fetch = item
+      @array = [hit]
+    end
+
     # @return [String]
     def group
       @group ||= case text
-                 when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR} then "Publications"
+                 when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR}
+                   "Publications"
                  when %r{^ITU-T} then "Recommendations"
                  end
     end
 
-    # rubocop:disable Metrics/MethodLength
-
     # @return [Hash]
-    def params
+    def params # rubocop:disable Metrics/MethodLength
       {
         "Input" => text,
         "Start" => 0,
@@ -102,7 +118,6 @@ module RelatonItu
         "SearchType" => "All",
       }
     end
-    # rubocop:enable Metrics/MethodLength
 
     # @param data [Hash]
     # @return [Array<RelatonItu::Hit>]

data/lib/relaton_itu/itu_bibliography.rb

@@ -28,13 +28,14 @@ module RelatonItu
         HitCollection.new text, year
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
-             Net::ProtocolError, OpenSSL::SSL::SSLError
-        raise RelatonBib::RequestError, "Could not access http://www.itu.int"
+             Net::ProtocolError, URI::InvalidURIError => e
+        raise RelatonBib::RequestError, e.message
       end
 
       # @param code [String] the ISO standard Code to look up (e..g "ISO 9000")
       # @param year [String] the year the standard was published (optional)
-      # @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
+      # @param opts [Hash] options; restricted to :all_parts if all-parts
+      #   reference is required
       # @return [String] Relaton XML serialisation of reference
       def get(code, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
         if year.nil?
@@ -61,12 +62,12 @@ module RelatonItu
         warn "[relaton-itu] WARNING: no match found online for #{id}. "\
           "The code must be exactly like it is on the standards website."
         unless missed_years.empty?
-          warn "[relaton-itu] (There was no match for #{year}, though there were matches "\
-            "found for #{missed_years.join(', ')}.)"
+          warn "[relaton-itu] (There was no match for #{year}, though there "\
+            "were matches found for #{missed_years.join(', ')}.)"
         end
         if /\d-\d/.match? code
-          warn "[relaton-itu] The provided document part may not exist, or the document "\
-            "may no longer be published in parts."
+          warn "[relaton-itu] The provided document part may not exist, or "\
+            "the document may no longer be published in parts."
         else
           warn "[relaton-itu] If you wanted to cite all document parts for the reference, "\
             "use \"#{code} (all parts)\".\nIf the document is not a standard, "\
@@ -77,10 +78,11 @@ module RelatonItu
 
       def search_filter(code, year) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
         %r{
-          ^(?<pref1>ITU)?(-(?<type1>\w))?\s?(?<code1>[^\s\/]+(?:\/\w\.\d+)?)
+          ^(?<pref1>ITU)?(-(?<type1>\w))?\s?(?<code1>[^\s\/]+(?:\/\w[\.\d]+)?)
+          (\s\(?(?<ver1>v\d+)\)?)?
           (\s\(((?<month1>\d{2})\/)?(?<year1>\d{4})\))?
           (\s-\s(?<buldate1>\d{2}\.\w{1,4}\.\d{4}))?
-          (\/(?<corr1>(Amd|Cor)\s?\d+))?
+          (\s(?<corr1>(Amd|Cor|Amendment|Corrigendum)\.?\s?\d+))?
           (\s\(((?<cormonth1>\d{2})\/)?(?<coryear1>\d{4})\))?
         }x =~ code
         year ||= year1
@@ -89,21 +91,27 @@ module RelatonItu
         warn "[relaton-itu] (\"#{code}\") fetching..."
         result = search(code)
         code1.sub! /(?<=\.)Imp(?=\d)/, "" if result.gi_imp
+        if corr1
+          corr1.sub!(/[\.\s]+/, " ").sub!("Amendment", "Amd")
+          corr1.sub!("Corrigendum", "Corr")
+        end
         result.select do |i|
           next true unless i.hit[:code]
 
           %r{
             ^(?<pref2>ITU)?(-(?<type2>\w))?\s?(?<code2>[\S]+)
+            (\s\(?(?<ver2>v\d+)\)?)?
             (\s\(((?<month2>\d{2})\/)?(?<year2>\d{4})\))?
             (\s(?<corr2>(Amd|Cor)\.\s?\d+))?
             (\s\(((?<cormonth2>\d{2})\/)?(?<coryear2>\d{4})\))?
           }x =~ i.hit[:code]
           /:[^\(]+\((?<buldate2>\d{2}\.\w{1,4}\.\d{4})\)/ =~ i.hit[:title]
           corr2&.sub! /\.\s?/, " "
-          pref1 == pref2 && (!type1 || type1 == type2) && code1 == code2 &&
+          pref1 == pref2 && (!type1 || type1 == type2) && code2.include?(code1) &&
             (!year || year == year2) && (!month1 || month1 == month2) &&
             corr1 == corr2 && (!coryear1 || coryear1 == coryear2) &&
-            buldate1 == buldate2 && (!cormonth1 || cormonth1 == cormonth2)
+            buldate1 == buldate2 && (!cormonth1 || cormonth1 == cormonth2) &&
+            (!ver1 || ver1 == ver2)
         end
       end
 
@@ -116,7 +124,8 @@ module RelatonItu
       def isobib_results_filter(result, year)
         missed_years = []
         result.each do |r|
-          if !year || /\((\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
+          /\((\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
+          if !year || year == pyear
             ret = r.fetch
             return { ret: ret } if ret
           end

data/lib/relaton_itu/scrapper.rb

@@ -24,20 +24,18 @@ module RelatonItu
     }.freeze
 
     class << self
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-
       # Parse page.
-      # @param hit_data [Hash]
+      # @param hit [RelatonItu::Hit]
       # @return [Hash]
-      def parse_page(hit_data, imp = false)
-        url, doc = get_page hit_data[:url]
-        return unless doc
+      def parse_page(hit, imp = false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        doc = get_page hit
+        return unless doc.code == "200"
 
         if imp
           a = doc.at "//span[contains(@id, 'tab_ig_uc_rec')]/a"
           return unless a
 
-          url, doc = get_page URI.join(url, a[:href]).to_s
+          doc = get_page hit, a[:href].to_s
         end
 
         # Fetch edition.
@@ -46,36 +44,37 @@ module RelatonItu
         ItuBibliographicItem.new(
           fetched: Date.today.to_s,
           type: "standard",
-          docid: fetch_docid(doc, hit_data[:title]),
+          docid: fetch_docid(doc, hit.hit[:title]),
           edition: edition,
           language: ["en"],
           script: ["Latn"],
           title: fetch_titles(doc),
-          doctype: hit_data[:type],
+          doctype: hit.hit[:type],
           docstatus: fetch_status(doc),
           ics: [], # fetch_ics(doc),
           date: fetch_dates(doc),
-          contributor: fetch_contributors(hit_data[:code]),
-          editorialgroup: fetch_workgroup(hit_data[:code], doc),
-          abstract: fetch_abstract(doc),
-          copyright: fetch_copyright(hit_data[:code], doc),
-          link: fetch_link(doc, url),
+          contributor: fetch_contributors(hit.hit[:code]),
+          editorialgroup: fetch_workgroup(hit.hit[:code], doc),
+          abstract: fetch_abstract(doc, hit),
+          copyright: fetch_copyright(hit.hit[:code], doc),
+          link: fetch_link(doc),
           relation: fetch_relations(doc),
           place: ["Geneva"]
         )
       end
-      # rubocop:enable Metrics/AbcSize
 
       private
 
       # Fetch abstracts.
-      # @param doc [Nokigiri::HTML::Document]
-      # @return [Array<Array>]
-      def fetch_abstract(doc) # rubocop:disable Metrics/AbcSize
-        abstract_url = doc.at('//table/tr/td/span[contains(@id, "lbl_dms")]/div')
+      # @param doc [Mechanize::Page]
+      # @param hit [RelatonItu::Hit]
+      # @return [Array<Hash>]
+      def fetch_abstract(doc, hit) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        abstract_url = doc.at '//table/tr[td/span[.="In force"]]/td/span[contains(@id, "lbl_dms")]/div'
         content = if abstract_url
                     url = abstract_url[:onclick].match(/https?[^']+/).to_s
-                    d = Nokogiri::HTML Net::HTTP.get(URI(url)).encode(undef: :replace, replace: "")
+                    rsp = hit.hit_collection.agent.get url
+                    d = Nokogiri::HTML rsp.body.encode(undef: :replace, replace: "")
                     d.css("p.MsoNormal").text.gsub(/\r\n/, "").squeeze(" ").gsub(/\u00a0/, "")
                   elsif a = doc.at('//table/tr/td/span[contains(@class, "observation")]/text()')
                     a.text.strip
@@ -90,27 +89,20 @@ module RelatonItu
       end
 
       # Get page.
-      # @param path [String] page's path
+      # @param hit [RelatonItu::Hit]
+      # @param url [String, nil]
       # @return [Array<String, Nokogiri::HTML::Document>]
-      def get_page(url)
-        uri = URI url
-        resp = Net::HTTP.get_response(uri)
-        until resp.code == "200"
-          return if resp["location"] == "/en/publications/pages/notfound.aspx"
-
-          uri = URI resp["location"] if resp.code.match? /^30/
-          resp = Net::HTTP.get_response(uri)
-        end
-        [uri.to_s, Nokogiri::HTML(resp.body)]
+      def get_page(hit, url = nil)
+        uri = url || hit.hit[:url]
+        hit.hit_collection.agent.get uri
       rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
              EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
              Net::ProtocolError, OpenSSL::SSL::SSLError
-        raise RelatonBib::RequestError, "Could not access #{url}"
+        raise RelatonBib::RequestError, "Could not access #{uri}"
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Fetch docid.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @param title [String]
       # @return [Hash]
       def fetch_docid(doc, title)
@@ -123,6 +115,8 @@ module RelatonItu
         docids
       end
 
+      # @param text [String]
+      # @return [RelatonBib::DocumentIdentifier]
       def createdocid(text) # rubocop:disable Metrics/MethodLength
         %r{
           ^(?<code>((ITU-\w|ISO\/IEC)\s)?[^\(:]+)
@@ -132,7 +126,7 @@ module RelatonItu
           # (\s\(((?<_cormonth>\d{2})\/)?(?<_coryear>\d{4})\))?
         }x =~ text.squeeze(" ")
         corr&.sub! /\.\s?/, " "
-        id = [code.sub(/[[:space:]]$/, ""), corr].compact.join "/"
+        id = [code.sub(/[[:space:]]$/, ""), corr].compact.join " "
         id += " - #{buldate}" if buldate
         type = id.match(%r{^\w+}).to_s
         type = "ITU" if type == "G"
@@ -140,7 +134,7 @@ module RelatonItu
       end
 
       # Fetch status.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [RelatonBib::DocumentStatus, NilClass]
       def fetch_status(doc)
         s = doc.at("//table/tr/td/span[contains(@id, 'Label7')]",
@@ -153,7 +147,7 @@ module RelatonItu
 
       # Fetch workgroup.
       # @param code [String]
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [RelatonItu::EditorialGroup, NilClass]
       def fetch_workgroup(code, doc)
         wg = doc.at('//table/tr/td/span[contains(@id, "Label8")]/a')
@@ -161,8 +155,7 @@ module RelatonItu
 
         group = wg && itugroup(wg.text)
         EditorialGroup.new(
-          bureau: code.match(/(?<=-)./).to_s,
-          group: group
+          bureau: code.match(/(?<=-)./).to_s, group: group
         )
       end
 
@@ -182,24 +175,24 @@ module RelatonItu
         ItuGroup.new name: name, type: type, acronym: acronym
       end
 
-      # rubocop:disable Metrics/MethodLength
-
       # Fetch relations.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_relations(doc)
-        doc.xpath('//div[contains(@id, "tab_sup")]//table/tr[position()>2]').map do |r|
+        doc.xpath('//div[contains(@id, "tab_sup")]//table/tr[position()>2]')
+          .map do |r|
           ref = r.at('./td/span[contains(@id, "title_e")]/nobr/a')
-          fref = RelatonBib::FormattedRef.new(content: ref.text, language: "en", script: "Latn")
-          bibitem = ItuBibliographicItem.new(formattedref: fref, type: "standard")
+          fref = RelatonBib::FormattedRef.new(content: ref.text, language: "en",
+                                              script: "Latn")
+          bibitem = ItuBibliographicItem.new(formattedref: fref,
+                                             type: "standard")
           { type: "complements", bibitem: bibitem }
         end
       end
-      # rubocop:enable Metrics/MethodLength
 
       # Fetch titles.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<Hash>]
+      # @param doc [Mechanize::Page]
+      # @return [RelatonBib::TypedTitleStringCollection]
       def fetch_titles(doc)
         t = doc.at("//td[@class='title']|//div/table[1]/tr[4]/td/strong")
         return [] unless t
@@ -208,7 +201,7 @@ module RelatonItu
       end
 
       # Fetch dates
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_dates(doc) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
         dates = []
@@ -224,7 +217,7 @@ module RelatonItu
       end
 
       # Scrape Operational Bulletin date.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [String]
       def ob_date(doc)
         pdate = doc.at('//table/tbody/tr/td[contains(text(), "Year:")]')
@@ -246,7 +239,7 @@ module RelatonItu
       end
 
       # Fetch contributors
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_contributors(code)
         return [] unless code
@@ -262,11 +255,10 @@ module RelatonItu
       end
 
       # Fetch links.
-      # @param doc [Nokogiri::HTML::Document]
-      # @param url [String]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
-      def fetch_link(doc, url)
-        links = [{ type: "src", content: url }]
+      def fetch_link(doc)
+        links = [{ type: "src", content: doc.uri.to_s }]
         obp_elm = doc.at(
           '//a[@title="Persistent link to download the PDF file"]',
           "//font[contains(.,'PDF')]/../.."
@@ -277,6 +269,8 @@ module RelatonItu
         links
       end
 
+      # @param type [String]
+      # @param elm [Nokogiri::XML::Element]
       def typed_link(type, elm)
         {
           type: type,
@@ -286,7 +280,7 @@ module RelatonItu
 
       # Fetch copyright.
       # @param code [String]
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Mechanize::Page]
       # @return [Array<Hash>]
       def fetch_copyright(code, doc)
         abbreviation = code.match(/^[^-]+/).to_s

data/lib/relaton_itu/version.rb

@@ -1,3 +1,3 @@
 module RelatonItu
-  VERSION = "1.7.6".freeze
+  VERSION = "1.8.0".freeze
 end

data/relaton-itu.gemspec

@@ -37,5 +37,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "vcr", "~> 5.0.0"
   spec.add_development_dependency "webmock"
 
-  spec.add_dependency "relaton-bib", "~> 1.7.0"
+  spec.add_dependency "mechanize", "~> 2.7.0"
+  spec.add_dependency "relaton-bib", "~> 1.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-itu
 version: !ruby/object:Gem::Version
-  version: 1.7.6
+  version: 1.8.0
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-04-12 00:00:00.000000000 Z
+date: 2021-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -122,20 +122,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.0
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: 1.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: 1.8.0
 description: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem
   model'
 email:
@@ -193,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem