relaton-itu 0.9.0 → 1.1.0

data/grammars/itu.rng

@@ -5,50 +5,10 @@
     we cannot have a new default namespace: we will end up with a grammar with two different
     namespaces, one for isostandard and one for csand additions. And we do not want that.
   -->
-  <include href="isostandard.rng">
+  <include href="isodoc.rng">
     <start>
       <ref name="itu"/>
     </start>
-    <define name="figure">
-      <element name="figure">
-        <attribute name="id">
-          <data type="ID"/>
-        </attribute>
-        <optional>
-          <ref name="tname"/>
-        </optional>
-        <choice>
-          <ref name="image"/>
-          <ref name="pre"/>
-          <oneOrMore>
-            <ref name="subfigure"/>
-          </oneOrMore>
-        </choice>
-        <zeroOrMore>
-          <ref name="fn"/>
-        </zeroOrMore>
-        <optional>
-          <ref name="dl"/>
-        </optional>
-        <zeroOrMore>
-          <ref name="note"/>
-        </zeroOrMore>
-      </element>
-    </define>
-    <define name="subfigure">
-      <element name="figure">
-        <attribute name="id">
-          <data type="ID"/>
-        </attribute>
-        <optional>
-          <ref name="tname"/>
-        </optional>
-        <choice>
-          <ref name="image"/>
-          <ref name="pre"/>
-        </choice>
-      </element>
-    </define>
     <define name="ol">
       <element name="ol">
         <attribute name="id">
@@ -97,11 +57,6 @@
         </optional>
       </element>
     </define>
-    <define name="workgroup">
-      <element name="workgroup">
-        <ref name="ItuGroup"/>
-      </element>
-    </define>
     <define name="structuredidentifier">
       <element name="structuredidentifier">
         <ref name="bureau"/>
@@ -147,13 +102,13 @@
         </oneOrMore>
       </element>
     </define>
-    <define name="TextElement" combine="choice">
-      <choice>
-        <ref name="add"/>
-        <ref name="del"/>
-      </choice>
-    </define>
   </include>
+  <define name="TextElement" combine="choice">
+    <choice>
+      <ref name="add"/>
+      <ref name="del"/>
+    </choice>
+  </define>
   <define name="add">
     <element name="add">
       <choice>
@@ -282,6 +237,11 @@
       <data type="gYear"/>
     </element>
   </define>
+  <define name="workgroup">
+    <element name="workgroup">
+      <ref name="ItuGroup"/>
+    </element>
+  </define>
   <define name="itu">
     <element name="itu-standard">
       <ref name="bibdata"/>

data/lib/relaton_itu.rb

@@ -2,11 +2,6 @@ require "relaton_itu/version"
 require "relaton_itu/itu_bibliography"
 require "digest/md5"
 
-# if defined? Relaton
-#   require_relative "relaton/processor"
-#   Relaton::Registry.instance.register(Relaton::RelatonItu::Processor)
-# end
-
 module RelatonItu
   class Error < StandardError; end
 

data/lib/relaton_itu/editorial_group.rb

@@ -16,8 +16,9 @@ module RelatonItu
     # @param subgroup [Hash, RelatonItu::ItuGroup, NilClass]
     # @param workgroup [Hash, RelatonItu::ItuGroup, NilClass]
     def initialize(bureau:, group:, subgroup: nil, workgroup: nil)
-      raise ArgumentError, "invalid bureau: #{bureau}" unless BUREAUS.include? bureau
-
+      unless BUREAUS.include? bureau
+        warn "[relaton-itu] WARNING: invalid bureau: #{bureau}"
+      end
       @bureau = bureau
       @group = group.is_a?(Hash) ? ItuGroup.new(group) : group
       @subgroup = subgroup.is_a?(Hash) ? ItuGroup.new(subgroup) : subgroup
@@ -28,7 +29,7 @@ module RelatonItu
     def to_xml(builder)
       builder.editorialgroup do
         builder.bureau bureau
-        builder.group { |b| group.to_xml b }
+        builder.group { |b| group.to_xml b } if group
         builder.subgroup { |b| group.to_xml b } if subgroup
         builder.workgroup { |b| group.to_xml b } if workgroup
       end
@@ -36,7 +37,8 @@ module RelatonItu
 
     # @return [Hash]
     def to_hash
-      hash = { "bureau" => bureau, "group" => group.to_hash }
+      hash = { "bureau" => bureau }
+      hash["group"] = group.to_hash if group
       hash["subgroup"] = subgroup.to_hash if subgroup
       hash["workgroup"] = workgroup.to_hash if workgroup
       hash

data/lib/relaton_itu/hash_converter.rb

@@ -9,6 +9,15 @@ module RelatonItu
 
         ret[:editorialgroup] = EditorialGroup.new eg
       end
+
+      # @param ret [Hash]
+      def structuredidentifier_hash_to_bib(ret)
+        return unless ret[:structuredidentifier]
+
+        ret[:structuredidentifier] = StructuredIdentifier.new(
+          ret[:structuredidentifier],
+        )
+      end
     end
   end
 end

data/lib/relaton_itu/hit.rb

@@ -4,9 +4,9 @@ module RelatonItu
   # Hit.
   class Hit < RelatonBib::Hit
     # Parse page.
-    # @return [Isobib::IsoBibliographicItem]
+    # @return [RelatonItu::ItuBibliographicItem]
     def fetch
-      @fetch ||= Scrapper.parse_page @hit
+      @fetch ||= Scrapper.parse_page hit, hit_collection.gi_imp
     end
   end
 end

data/lib/relaton_itu/hit_collection.rb

@@ -7,16 +7,39 @@ require "net/http"
 module RelatonItu
   # Page of hit collection.
   class HitCollection < RelatonBib::HitCollection
-    DOMAIN = "https://www.itu.int".freeze
+    DOMAIN = "https://www.itu.int"
 
-    # @param ref_nbr [String]
+    # @return [TrueClass, FalseClass]
+    attr_reader :gi_imp
+
+    # @param ref [String]
     # @param year [String]
-    def initialize(ref_nbr, year = nil)
-      super
-      group = %r{(OB|Operational Bulletin) No} =~ text ? "Publications" : "Recommendations"
-      url = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
-      params = {
-        "Input" => ref_nbr,
+    def initialize(ref, year = nil)
+      text = ref.sub /(?<=\.)Imp\s?(?=\d)/, ""
+      super text, year
+      @gi_imp = /\.Imp\d/.match?(ref)
+      uri = URI "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
+      data = { json: params.to_json }
+      resp = Net::HTTP.post(uri, data.to_json,
+                            "Content-Type" => "application/json")
+      @array = hits JSON.parse(resp.body)
+    end
+
+    private
+
+    # @return [String]
+    def group
+      @group ||= if %r{(OB|Operational Bulletin) No} =~ text then "Publications"
+                 else "Recommendations"
+                 end
+    end
+
+    # rubocop:disable Metrics/MethodLength
+
+    # @return [Hash]
+    def params
+      {
+        "Input" => text,
         "Start" => 0,
         "Rows" => 10,
         "SortBy" => "RELEVANCE",
@@ -61,10 +84,13 @@ module RelatonItu
         "IP" => "",
         "SearchType" => "All",
       }
-      data = { json: params.to_json }
-      resp  = Net::HTTP.post(URI(url), data.to_json, "Content-Type" => "application/json")
-      doc = JSON.parse resp.body
-      @array = doc["results"].map do |h|
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    # @param data [Hash]
+    # @return [Array<RelatonItu::Hit>]
+    def hits(data)
+      data["results"].map do |h|
         code  = h["Media"]["Name"]
         title = h["Title"]
         url   = h["Redirection"]

data/lib/relaton_itu/itu_bibliographic_item.rb

@@ -7,12 +7,12 @@ module RelatonItu
       joint-itu-iso-iec
     ].freeze
 
+    # @params structuredidentifier [RelatonItu::StructuredIdentifier]
     def initialize(**args)
       @doctype = args.delete :doctype
       if doctype && !TYPES.include?(doctype)
-        raise ArgumentError, "invalid doctype: #{doctype}"
+        warn "[relaton-itu] WARNING: invalid doctype: #{doctype}"
       end
-
       super
     end
   end

data/lib/relaton_itu/itu_bibliography.rb

@@ -3,6 +3,7 @@
 require "relaton_iso_bib"
 require "relaton_itu/itu_bibliographic_item"
 require "relaton_itu/editorial_group"
+require "relaton_itu/structured_identifier"
 require "relaton_itu/itu_group"
 require "relaton_itu/scrapper"
 require "relaton_itu/hit_collection"
@@ -19,9 +20,9 @@ module RelatonItu
       # @return [RelatonItu::HitCollection]
       def search(text, year = nil)
         HitCollection.new text, year
-      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
-             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
-             OpenSSL::SSL::SSLError
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, OpenSSL::SSL::SSLError
         raise RelatonBib::RequestError, "Could not access http://www.itu.int"
       end
 
@@ -66,17 +67,17 @@ module RelatonItu
         nil
       end
 
-      def fetch_pages(s, n)
-        workers = RelatonBib::WorkersPool.new n
-        workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
-        s.each_with_index { |hit, i| workers << { i: i, hit: hit } }
-        workers.end
-        workers.result.sort { |x, y| x[:i] <=> y[:i] }.map { |x| x[:hit] }
-      end
+      # def fetch_pages(hits, threads)
+      #   workers = RelatonBib::WorkersPool.new threads
+      #   workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
+      #   hits.each_with_index { |hit, i| workers << { i: i, hit: hit } }
+      #   workers.end
+      #   workers.result.sort_by { |a| a[:i] }.map { |x| x[:hit] }
+      # end
 
       def search_filter(code)
-        docidrx = %r{\w+.\d+|\w\sSuppl\.\s\d+} # %r{^ITU-T\s[^\s]+}
-        c = code.match(docidrx).to_s
+        docidrx = %r{\w+\.\d+|\w\sSuppl\.\s\d+} # %r{^ITU-T\s[^\s]+}
+        c = code.sub(/Imp\s?/, "").match(docidrx).to_s
         warn "[relaton-itu] (\"#{code}\") fetching..."
         result = search(code)
         result.select do |i|
@@ -93,16 +94,18 @@ module RelatonItu
       # If no match, returns any years which caused mismatch, for error reporting
       def isobib_results_filter(result, year)
         missed_years = []
-        result.each_slice(3) do |s| # ISO website only allows 3 connections
-          fetch_pages(s, 3).each do |r|
-            return { ret: r } if !year
+        # result.each_slice(3) do |s| # ISO website only allows 3 connections
+        #   fetch_pages(s, 3).each do |r|
+        result.each do |r|
+          return { ret: r.fetch } if !year
 
-            r.date.select { |d| d.type == "published" }.each do |d|
-              return { ret: r } if year.to_i == d.on.year
+          /\(\d{2}\/(?<pyear>\d{4})\)/ =~ r.hit[:code]
+          # r.date.select { |d| d.type == "published" }.each do |d|
+          return { ret: r.fetch } if year == pyear
 
-              missed_years << d.on.year
-            end
-          end
+          missed_years << pyear
+          # end
+          # end
         end
         { years: missed_years }
       end

data/lib/relaton_itu/scrapper.rb

@@ -3,16 +3,9 @@
 require "nokogiri"
 require "net/http"
 
-# Capybara.request_driver :poltergeist do |app|
-#   Capybara::Poltergeist::Driver.new app, js_errors: false
-# end
-# Capybara.default_driver = :poltergeist
-
 module RelatonItu
   # Scrapper.
-  # rubocop:disable Metrics/ModuleLength
   module Scrapper
-    DOMAIN = "https://www.itu.int"
     ROMAN_MONTHS = %w[I II III IV V VI VII VIII IX X XI XII].freeze
 
     TYPES = {
@@ -31,24 +24,19 @@ module RelatonItu
     }.freeze
 
     class << self
-      # @param text [String]
-      # @return [Array<Hash>]
-      # def get(text)
-      #   iso_workers = WorkersPool.new 4
-      #   iso_workers.worker { |hit| iso_worker(hit, iso_workers) }
-      #   algolia_workers = start_algolia_search(text, iso_workers)
-      #   iso_docs = iso_workers.result
-      #   algolia_workers.end
-      #   algolia_workers.result
-      #   iso_docs
-      # end
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
 
       # Parse page.
-      # @param hit [Hash]
+      # @param hit_data [Hash]
       # @return [Hash]
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      def parse_page(hit_data)
+      def parse_page(hit_data, imp = false)
         url, doc = get_page hit_data[:url]
+        if imp
+          a = doc.at "//span[contains(@id, 'tab_ig_uc_rec')]/a"
+          return unless a
+
+          url, doc = get_page URI.join(url, a[:href]).to_s
+        end
 
         # Fetch edition.
         edition = doc.at("//table/tr/td/span[contains(@id, 'Label8')]/b")&.text
@@ -73,7 +61,7 @@ module RelatonItu
           place: ["Geneva"],
         )
       end
-      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+      # rubocop:enable Metrics/AbcSize
 
       private
 
@@ -96,37 +84,23 @@ module RelatonItu
         }]
       end
 
-      # Get langs.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<Hash>]
-      # def langs(doc)
-      #   lgs = [{ lang: 'en' }]
-      #   doc.css('ul#lang-switcher ul li a').each do |lang_link|
-      #     lang_path = lang_link.attr('href')
-      #     lang = lang_path.match(%r{^\/(fr)\/})
-      #     lgs << { lang: lang[1], path: lang_path } if lang
-      #   end
-      #   lgs
-      # end
-
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       # Get page.
       # @param path [String] page's path
-      # @return [Array<Nokogiri::HTML::Document, String>]
+      # @return [Array<String, Nokogiri::HTML::Document>]
       def get_page(url)
         uri = URI url
-        resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
+        resp = Net::HTTP.get_response(uri)
         until resp.code == "200"
           uri = URI resp["location"] if resp.code =~ /^30/
-          resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
+          resp = Net::HTTP.get_response(uri)
         end
         [uri.to_s, Nokogiri::HTML(resp.body)]
-      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
-             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
-             OpenSSL::SSL::SSLError
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, OpenSSL::SSL::SSLError
         raise RelatonBib::RequestError, "Could not access #{url}"
       end
-      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+      # rubocop:enable Metrics/MethodLength
 
       # Fetch docid.
       # @param doc [Nokogiri::HTML::Document]
@@ -135,9 +109,11 @@ module RelatonItu
         doc.xpath(
           "//span[@id='ctl00_content_main_uc_rec_main_info1_rpt_main_ctl00_lbl_rec']",
           "//td[.='Identical standard:']/following-sibling::td",
+          "//div/table[1]/tr[4]/td/strong",
         ).map do |code|
-          id = code.text.match(%r{^.*?(?= \()}).to_s.squeeze(" ")
+          id = code.text.match(%r{^.*?(?= \()|\w\.Imp\s?\d+}).to_s.squeeze(" ")
           type = id.match(%r{^\w+}).to_s
+          type = "ITU" if type == "G"
           RelatonBib::DocumentIdentifier.new(type: type, id: id)
         end
       end
@@ -146,10 +122,11 @@ module RelatonItu
       # @param doc [Nokogiri::HTML::Document]
       # @return [RelatonBib::DocumentStatus, NilClass]
       def fetch_status(doc)
-        s = doc.at("//table/tr/td/span[contains(@id, 'Label7')]")
+        s = doc.at("//table/tr/td/span[contains(@id, 'Label7')]",
+                   "//p[contains(.,'Status :')]")
         return unless s
 
-        status = s.text == "In force" ? "Published" : "Withdrawal"
+        status = s.text.include?("In force") ? "Published" : "Withdrawal"
         RelatonBib::DocumentStatus.new(stage: status)
       end
 
@@ -191,9 +168,7 @@ module RelatonItu
       # @return [Array<Hash>]
       def fetch_relations(doc)
         doc.xpath('//div[contains(@id, "tab_sup")]//table/tr[position()>2]').map do |r|
-          # r_type = r.at('./td/span[contains(@id, "Label4")]/nobr').text.downcase
           ref = r.at('./td/span[contains(@id, "title_e")]/nobr/a')
-          # url = DOMAIN + ref[:href].sub(/^\./, "/ITU-T/recommendations")
           fref = RelatonBib::FormattedRef.new(content: ref.text, language: "en", script: "Latn")
           bibitem = RelatonIsoBib::IsoBibliographicItem.new(formattedref: fref)
           { type: "complements", bibitem: bibitem }
@@ -201,22 +176,14 @@ module RelatonItu
       end
       # rubocop:enable Metrics/MethodLength
 
-      # Fetch type.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [String]
-      # def fetch_type(_doc)
-      #   "recommendation"
-      # end
-
       # Fetch titles.
       # @param doc [Nokogiri::HTML::Document]
       # @return [Array<Hash>]
       def fetch_titles(doc)
-        # t = hit_data[:title].match(%r{(?<=\(\d{2}\/\d{4}\): ).*}).to_s
-        # t = hit_data[:title] if t.empty?
-        t = doc.at("//td[@class='title']")
+        t = doc.at("//td[@class='title']|//div/table[1]/tr[4]/td/strong")
         return [] unless t
-        titles = t.text.split " - "
+
+        titles = t.text.sub(/\w\.Imp\s?\d+\u00A0:\u00A0/, "").split " - "
         case titles.size
         when 0
           intro, main, part = nil, "", nil
@@ -247,10 +214,11 @@ module RelatonItu
       # @return [Array<Hash>]
       def fetch_dates(doc)
         dates = []
-        pdate = doc.at("//table/tr/td/span[contains(@id, 'Label5')]")
-        publish_date = pdate&.text || ob_date(doc)
-        if publish_date && !publish_date&.empty?
-          dates << { type: "published", on: publish_date }
+        date = doc.at("//table/tr/td/span[contains(@id, 'Label5')]",
+                      "//p[contains(.,'Approved in')]")
+        pdate = date&.text&.match(/\d{4}-\d{2}-\d{2}/).to_s || ob_date(doc)
+        if pdate && !pdate&.empty?
+          dates << { type: "published", on: pdate }
         end
         dates
       end
@@ -278,40 +246,45 @@ module RelatonItu
       # @param doc [Nokogiri::HTML::Document]
       # @return [Array<Hash>]
       def fetch_contributors(code)
+        return [] unless code
+
         abbrev = code.sub(/-\w\s.*/, "")
         case abbrev
         when "ITU"
           name = "International Telecommunication Union"
           url = "www.itu.int"
         end
-        [{ entity: { name: name, url: url, abbreviation: abbrev }, role: [type: "publisher"] }]
+        [{ entity: { name: name, url: url, abbreviation: abbrev },
+           role: [type: "publisher"] }]
       end
 
-      # Fetch ICS.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<Hash>]
-      # def fetch_ics(doc)
-      #   doc.xpath('//th[contains(text(), "ICS")]/following-sibling::td/a').map do |i|
-      #     code = i.text.match(/[\d\.]+/).to_s.split '.'
-      #     { field: code[0], group: code[1], subgroup: code[2] }
-      #   end
-      # end
-
       # Fetch links.
       # @param doc [Nokogiri::HTML::Document]
       # @param url [String]
       # @return [Array<Hash>]
       def fetch_link(doc, url)
         links = [{ type: "src", content: url }]
-        obp_elms = doc.at('//a[@title="Persistent link to download the PDF file"]')
-        links << { type: "obp", content: DOMAIN + obp_elms[:href].strip } if obp_elms
+        obp_elm = doc.at(
+          '//a[@title="Persistent link to download the PDF file"]',
+          "//font[contains(.,'PDF')]/../..",
+        )
+        links << typed_link("obp", obp_elm) if obp_elm
+        wrd_elm = doc.at("//font[contains(.,'Word')]/../..")
+        links << typed_link("word", wrd_elm) if wrd_elm
         links
       end
 
+      def typed_link(type, elm)
+        {
+          type: type,
+          content: URI.join(HitCollection::DOMAIN + elm[:href].strip).to_s,
+        }
+      end
+
       # Fetch copyright.
       # @param code [String]
       # @param doc [Nokogiri::HTML::Document]
-      # @return [Hash]
+      # @return [Array<Hash>]
       def fetch_copyright(code, doc)
         abbreviation = code.match(/^[^-]+/).to_s
         case abbreviation
@@ -321,9 +294,9 @@ module RelatonItu
         end
         fdate = doc.at("//table/tr/td/span[contains(@id, 'Label5')]")
         from = fdate&.text || ob_date(doc)
-        { owner: { name: name, abbreviation: abbreviation, url: url }, from: from }
+        [{ owner: [{ name: name, abbreviation: abbreviation, url: url }],
+           from: from }]
       end
     end
   end
-  # rubocop:enable Metrics/ModuleLength
 end