relaton-un 0.2.0 → 1.2.0

data/grammars/un.rng

@@ -107,7 +107,9 @@
       <zeroOrMore>
         <ref name="submissionlanguage"/>
       </zeroOrMore>
-      <ref name="editorialgroup"/>
+      <optional>
+        <ref name="editorialgroup"/>
+      </optional>
       <zeroOrMore>
         <ref name="ics"/>
       </zeroOrMore>
@@ -117,6 +119,9 @@
       <optional>
         <ref name="session"/>
       </optional>
+      <optional>
+        <ref name="job_number"/>
+      </optional>
     </define>
     <define name="preface">
       <element name="preface">
@@ -136,6 +141,109 @@
         <ref name="Basic-Section"/>
       </element>
     </define>
+    <define name="Clause-Section">
+      <optional>
+        <attribute name="id">
+          <data type="ID"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="language"/>
+      </optional>
+      <optional>
+        <attribute name="script"/>
+      </optional>
+      <optional>
+        <attribute name="inline-header">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="obligation">
+          <choice>
+            <value>normative</value>
+            <value>informative</value>
+          </choice>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="unnumbered">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
+      <optional>
+        <ref name="section-title"/>
+      </optional>
+      <group>
+        <group>
+          <zeroOrMore>
+            <ref name="BasicBlock"/>
+          </zeroOrMore>
+          <zeroOrMore>
+            <ref name="note"/>
+          </zeroOrMore>
+        </group>
+        <zeroOrMore>
+          <choice>
+            <ref name="clause-subsection"/>
+            <ref name="terms"/>
+            <ref name="definitions"/>
+          </choice>
+        </zeroOrMore>
+      </group>
+    </define>
+    <define name="Annex-Section">
+      <optional>
+        <attribute name="id">
+          <data type="ID"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="language"/>
+      </optional>
+      <optional>
+        <attribute name="script"/>
+      </optional>
+      <optional>
+        <attribute name="inline-header">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="obligation">
+          <choice>
+            <value>normative</value>
+            <value>informative</value>
+          </choice>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="unnumbered">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
+      <optional>
+        <ref name="section-title"/>
+      </optional>
+      <group>
+        <group>
+          <zeroOrMore>
+            <ref name="BasicBlock"/>
+          </zeroOrMore>
+          <zeroOrMore>
+            <ref name="note"/>
+          </zeroOrMore>
+        </group>
+        <zeroOrMore>
+          <choice>
+            <ref name="annex-subsection"/>
+            <ref name="terms"/>
+            <ref name="definitions"/>
+            <ref name="references"/>
+          </choice>
+        </zeroOrMore>
+      </group>
+    </define>
   </include>
   <define name="session">
     <element name="session">
@@ -221,9 +329,15 @@
         <value>general</value>
         <value>limited</value>
         <value>restricted</value>
+        <value>provisional</value>
       </choice>
     </element>
   </define>
+  <define name="job_number">
+    <element name="job_number">
+      <text/>
+    </element>
+  </define>
   <define name="un-standard">
     <element name="un-standard">
       <ref name="bibdata"/>

data/lib/relaton_un.rb

@@ -6,6 +6,8 @@ require "relaton_un/hit_collection"
 require "relaton_un/hit"
 require "relaton_un/hash_converter"
 require "relaton_un/xml_parser"
+require "relaton_un/session"
+require "relaton_un/editorialgroup"
 
 module RelatonUn
   class Error < StandardError; end

data/lib/relaton_un/editorialgroup.rb

@@ -0,0 +1,25 @@
+module RelatonUn
+  class EditorialGroup
+    include RelatonBib
+
+    # @return [Array<String>]
+    attr_reader :committee
+
+    # @param committee [Array<String>]
+    def initialize(committee)
+      @committee = committee
+    end
+
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(builder)
+      builder.editorialgroup do |b|
+        committee.each { |c| b.committee c }
+      end
+    end
+
+    # @return [Array<Hash>, Hash]
+    def to_hash
+      single_element_array(committee.map { |c| { "committee" => c } })
+    end
+  end
+end

data/lib/relaton_un/hash_converter.rb

@@ -1,5 +1,33 @@
 module RelatonUn
   class HashConverter < RelatonBib::HashConverter
+    class << self
+      # @override RelatonIsoBib::HashConverter.hash_to_bib
+      # @param args [Hash]
+      # @param nested [TrueClass, FalseClass]
+      # @return [Hash]
+      def hash_to_bib(args, nested = false)
+        ret = super
+        return if ret.nil?
 
+        session_hash_to_bib ret
+        ret
+      end
+
+      private
+
+      # @param ret [Hash]
+      def session_hash_to_bib(ret)
+        ret[:session] = Session.new(ret[:session]) if ret[:session]
+      end
+
+      # @param ret [Hash]
+      def editorialgroup_hash_to_bib(ret)
+        eg = ret[:editorialgroup]
+        return unless eg
+
+        committee = eg.map { |e| e[:committee] }
+        ret[:editorialgroup] = EditorialGroup.new array(committee)
+      end
+    end
   end
 end

data/lib/relaton_un/hit.rb

@@ -3,6 +3,51 @@
 module RelatonUn
   # Hit.
   class Hit < RelatonBib::Hit
+    # rubocop:disable Layout/LineLength
+
+    # There is distribution PRO (A/47/PV.102/CORR.1, A/47/PV.54)
+    BODY = {
+      "A" => "General Assembly",
+      "E" => "Economic and Social Council",
+      "S" => "Security Council",
+      "T" => "Trusteeship Council",
+      "ACC" => "Administrative Committee on Coordination",
+      "AT" => "United Nations Administrative Tribunal",
+      "CAT" => "Committee against Torture",
+      "CCPR" => "Human Rights Committee",
+      "CD" => "Conference on Disarmament",
+      "CEDAW" => "Committee on the Elimination of All Forms of Discrimination against Women",
+      "CERD" => "Committee on the Elimination of Racial Discrimination",
+      "CRC" => "Committee on the Rights of the Child",
+      "DC" => "Disarmament Commission",
+      "DP" => "United Nations Development Programme",
+      "HS" => "United Nations Centre for Human Settlements (HABITAT)",
+      "TD" => "United Nations Conference on Trade and Development",
+      "UNEP" => "United Nations Environment Programme",
+      "TRADE" => "Committee on Trade",
+      "CEFACT" => "Centre for Trade Facilitation and Electronic Business",
+      "C.1" => "Disarmament and International Security Committee",
+      "C.2" => "Economic and Financial Committee",
+      "C.3" => "Social, Humanitarian & Cultural Issues",
+      "C.4" => "Special Political and Decolonization Committee",
+      "C.5" => "Administrative and Budgetary Committee",
+      "C.6" => "Sixth Committee (Legal)",
+      "PC" => "Preparatory Committee",
+      "AEC" => "Atomic Energy Commission",
+      "AGRI" => "Committee on Agriculture",
+      "AMCEN" => "African Ministerial Conference on the Environment",
+      "AMCOW" => "African Ministers’ Council on Water",
+      "ECA" => "Economic Commission for Africa",
+      "ESCAP" => "Economic and Social Commission for Asia and Pacific",
+      "ECE" => "Economic Commission for Europe",
+      "ECWA" => "Economic Commission for Western Asia",
+      "UNFF" => "United Nations Forum on Forests",
+      "ENERGY" => "Committee on Sustainable Energy",
+      "FAO" => "Food and Agriculture Organization",
+      "UNCTAD" => "United Nations Conference on Trade and Development",
+    }.freeze
+    # rubocop:enable Layout/LineLength
+
     # Parse page.
     # @return [RelatonUn::UnBibliographicItem]
     def fetch
@@ -11,48 +56,97 @@ module RelatonUn
 
     private
 
+    # rubocop:disable Metrics/MethodLength
+
+    # @return [RelatonUn::UnBibliographicItem]
     def un_bib_item
       UnBibliographicItem.new(
         type: "standard",
         fetched: Date.today.to_s,
-        docid: docid,
+        docid: fetch_docid,
         docnumber: hit[:ref],
         language: ["en"],
         script: ["Latn"],
-        title: title,
-        date: date,
-        link: link,
-        keyword: keyword
+        title: fetch_title,
+        date: fetch_date,
+        link: fetch_link,
+        keyword: fetch_keyword,
+        session: fetch_session,
+        distribution: fetch_distribution,
+        editorialgroup: fetch_editorialgroup,
+        classification: fetch_classification,
       )
     end
+    # rubocop:enable Metrics/MethodLength
 
     # @return [Array<RelatonBib::DocumentIdentifier>]
-    def docid
-      [RelatonBib::DocumentIdentifier.new(id: hit[:ref], type: "UN")]
+    def fetch_docid
+      hit[:symbol].map do |s|
+        RelatonBib::DocumentIdentifier.new(id: s, type: "UN")
+      end
     end
 
     # @return [Array<RelatonBib::TypedTitleString>]
-    def title
-      fs = RelatonBib::FormattedString.new(content: hit[:title], language: "en", script: "Latn")
-      [RelatonBib::TypedTitleString.new(type: "main", title: fs)]
+    def fetch_title
+      # fs = RelatonBib::FormattedString.new(
+      #   content: hit[:title], language: "en", script: "Latn",
+      # )
+      # [RelatonBib::TypedTitleString.new(type: "main", title: fs)]
+      # [{ title_main: hit[:title], language: "en", script: "Latn" }]
+      RelatonBib::TypedTitleString.from_string hit[:title], "en", "Latn"
     end
 
     # @return [Array<RelatonBib::BibliographicDate>]
-    def date
+    def fetch_date
       d = []
-      d << RelatonBib::BibliographicDate.new(type: "published", on: hit[:date_pub]) if hit[:date_pub]
-      d << RelatonBib::BibliographicDate.new(type: "issued", on: hit[:date_rel]) if hit[:date_rel]
+      d << bibdate("published", hit[:date_pub]) if hit[:date_pub]
+      d << bibdate("issued", hit[:date_rel]) if hit[:date_rel]
       d
     end
 
+    # @param type [String]
+    # @param on [String]
+    # @return [RelatonBib::BibliographicDate]
+    def bibdate(type, on)
+      RelatonBib::BibliographicDate.new type: type, on: on
+    end
+
     # @return [Array<RelatonBib::TypedUri>]
-    def link
+    def fetch_link
       hit[:link].map { |l| RelatonBib::TypedUri.new l }
     end
 
     # @return [Array<String>]
-    def keyword
+    def fetch_keyword
       hit[:keyword].split(", ")
     end
+
+    # @return [RelatonUn::Session]
+    def fetch_session
+      Session.new(session_number: hit[:session], agenda_id: hit[:agenda])
+    end
+
+    # @return [String]
+    def fetch_distribution
+      UnBibliographicItem::DISTRIBUTIONS[hit[:distribution]]
+    end
+
+    # @return [RelatonUn::EditorialGroup, NilClass]
+    def fetch_editorialgroup
+      tc = hit[:ref].match(/^[\S]+/).to_s.split(/\/|-/).reduce([]) do |m, v|
+        if BODY[v] then m << BODY[v]
+        elsif v =~ /(AC|C|CN|CONF|GC|SC|Sub|WG).\d+|PC/ then m << v
+        else m
+        end
+      end.uniq
+      return unless tc.any?
+
+      RelatonUn::EditorialGroup.new tc
+    end
+
+    # @return [Array<RelatonBib::Classification>]
+    def fetch_classification
+      [RelatonBib::Classification.new(type: "area", value: "UNDOC")]
+    end
   end
 end

data/lib/relaton_un/hit_collection.rb

@@ -6,7 +6,8 @@ require "http-cookie"
 module RelatonUn
   # Page of hit collection.
   class HitCollection < RelatonBib::HitCollection
-    AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"
+    AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) "\
+    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"
     DOMAIN = "https://documents.un.org"
     BOUNDARY = "----WebKitFormBoundary6hkaBvITDck8dHCn"
 
@@ -17,52 +18,10 @@ module RelatonUn
       @jar = HTTP::CookieJar.new
       @http = Net::HTTP.new @uri.host, @uri.port
       @http.use_ssl = true
+      @http.read_timeout = 120
       if (form_resp = get_page)
-        form = Nokogiri::HTML form_resp.body
-        form_data = form.xpath(
-          "//input[@type!='radio']",
-          "//input[@type='radio'][@checked]",
-          "//select[@name!='view:_id1:_id2:cbLang']",
-          "//textarea"
-        ).reduce([]) do |m, i|
-          v = case i[:name]
-              when "view:_id1:_id2:txtSymbol" then text
-              when "view:_id1:_id2:cbType" then "FP"
-              when "view:_id1:_id2:cbSort" then "R"
-              when "$$xspsubmitid" then "view:_id1:_id2:_id130"
-              when "$$xspsubmitscroll" then "0|167"
-              else i[:value]
-              end
-          m << %{--#{BOUNDARY}}
-          m << %{Content-Disposition: form-data; name="#{i[:name]}"\r\n\r\n#{v}}
-        end
-        form_data << %{--#{BOUNDARY}--\r\n}
-        req = Net::HTTP::Post.new form.at("//form")[:action]
-        set_headers req
-        req["Content-Type"] = "multipart/form-data, boundary=#{BOUNDARY}"
-        req.body = form_data.join("\r\n")
-        resp = @http.request req
-        page_resp = get_page URI.parse(resp["location"]).request_uri
-        doc = Nokogiri::HTML page_resp.body
-        @array = doc.css("div.viewHover").map do |item|
-          ref = item.at("div/div/a")&.text&.sub "\u00A0", ""
-          title = item.at("div/div/span")&.text
-          keyword = item.at("div[3]/div[5]/span")&.text
-          date_pub = item.at("//label[.='Publication Date: ']/following-sibling::span")&.text
-          en = item.at("//span[.='ENGLISH']/../..")
-          date_rel = en.at("./following-sibling::span[contains(@id, 'cfRelDateE')]").text
-          link = en.xpath("//a[contains(@title, 'Open')]").map do |l|
-            { content: l[:href], type: l[:title].match(/PDF|Word/).to_s.downcase }
-          end
-          Hit.new({
-            ref: ref,
-            title: title,
-            keyword: keyword,
-            date_pub: date_pub,
-            date_rel: date_rel,
-            link: link
-          }, self)
-        end
+        doc = Nokogiri::HTML page_resp(form_resp, text).body
+        @array = doc.css("div.viewHover").map { |item| hit item }
       end
     end
 
@@ -84,9 +43,125 @@ module RelatonUn
       get_page request_uri, deep + 1
     end
 
+    # rubocop:disable Metrics/MethodLength
+
+    # @param form [Nokogiri::HTML::Document]
+    # @param text [String]
+    # @return [Array<String>]
+    def form_data(form, text)
+      fd = form.xpath(
+        "//input[@type!='radio']",
+        "//input[@type='radio'][@checked]",
+        "//select[@name!='view:_id1:_id2:cbLang']",
+        "//textarea",
+      ).reduce([]) do |m, i|
+        v = case i[:name]
+            when "view:_id1:_id2:txtSymbol" then text
+            when "view:_id1:_id2:cbType" then "FP"
+            when "view:_id1:_id2:cbSort" then "R"
+            when "$$xspsubmitid" then "view:_id1:_id2:_id130"
+            when "$$xspsubmitscroll" then "0|167"
+            else i[:value]
+            end
+        m << %{--#{BOUNDARY}}
+        m << %{Content-Disposition: form-data; name="#{i[:name]}"\r\n\r\n#{v}}
+      end
+      fd << %{--#{BOUNDARY}--\r\n}
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    # @param form_resp [Net::HTTPOK]
+    # @param text [String]
+    # @return [Net::HTTPOK]
+    def page_resp(form_resp, text)
+      form = Nokogiri::HTML form_resp.body
+      req = Net::HTTP::Post.new form.at("//form")[:action]
+      set_headers req
+      req["Content-Type"] = "multipart/form-data, boundary=#{BOUNDARY}"
+      req.body = form_data(form, text).join("\r\n")
+      resp = @http.request req
+      get_page URI.parse(resp["location"]).request_uri
+    end
+
+    # @param item [Nokogiri::XML::Element]
+    # @return [RelatonUn::Hit]
+    def hit(item)
+      Hit.new(hit_data(item), self)
+    end
+
+    # @param item [Nokogiri::XML::Element]
+    # @return [Hash]
+    def hit_data(item)
+      en = item.at("//span[.='ENGLISH']/../..")
+      {
+        ref: item.at("div/div/a")&.text&.sub("\u00A0", ""),
+        symbol: symbol(item),
+        title: item.at("div/div/span")&.text,
+        keyword: item.at("div[3]/div[5]/span")&.text,
+        date_pub: date_pub(item),
+        date_rel: date_rel(en),
+        link: link(en),
+        session: session(item),
+        agenda: agenda(item),
+        distribution: distribution(item)
+      }
+    end
+
+    # @param item [Nokogiri::XML::Element]
+    # @return [String]
+    def symbol(item)
+      item.xpath("div/div[not(contains(@class, 'hidden'))]/"\
+        "label[contains(.,'Symbol')]/following-sibling::span[1]").map &:text
+    end
+
+    # @param item [Nokogiri::XML::Element]
+    # @return [String]
+    def date_pub(item)
+      item.at("//label[.='Publication Date: ']/following-sibling::span")&.text
+    end
+
+    # @param item [Nokogiri::XML::Element]
+    # @return [String]
+    def date_rel(item)
+      item.at("./following-sibling::span[contains(@id, 'cfRelDateE')]")&.text
+    end
+
+    # @param item [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def link(item)
+      item.xpath("//a[contains(@title, 'Open')]").map do |l|
+        {
+          content: l[:href],
+          type: l[:title].match(/PDF|Word/).to_s.downcase,
+        }
+      end
+    end
+
+    # @param item [Nokogiri::XML::Element]
+    # @return [String]
+    def session(item)
+      item.at("//label[.='Session / Year:']/following-sibling::span")&.text
+    end
+
+    # @param item [Nokogiri::XML::Element]
+    # @return [String]
+    def agenda(item)
+      item.at("//label[.='Agenda Item(s):']/following-sibling::span")&.text
+    end
+
+    # @param item [Nokogiri::XML::Element]
+    # @return [String]
+    def distribution(item)
+      item.at("//label[.='Distribution:']/following-sibling::span")&.text
+    end
+
+    # rubocop:disable Metrics/MethodLength
+
+    # @param req [Net::HTTP::Get, Net::HTTP::Post]
     def set_headers(req)
       set_cookie req
-      req["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
+      req["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,"\
+      "image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
       req["Accept-Encoding"] = "gzip, deflate, br"
       req["Cache-Control"] = "max-age=0"
       req["Connection"] = "keep-alive"
@@ -98,7 +173,9 @@ module RelatonUn
       req["Upgrade-Insecure-Requests"] = "1"
       req["User-Agent"] = AGENT
     end
+    # rubocop:enable Metrics/MethodLength
 
+    # @param req [Net::HTTP::Get, Net::HTTP::Post]
     def set_cookie(req)
       req["Cookie"] = HTTP::Cookie.cookie_value @jar.cookies(@uri)
     end