relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/iso.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "pubid"
+require "relaton/index"
+require "isoics"
+require "relaton/bib"
+require "relaton/core"
+require_relative "version"
+require_relative "iso/util"
+require_relative "iso/item_data"
+require_relative "iso/model/item"
+require_relative "iso/model/relation"
+require_relative "iso/model/bibitem"
+require_relative "iso/model/bibdata"
+require_relative "iso/hit_collection"
+require_relative "iso/bibliography"
+
+module Relaton
+  module Iso
+    INDEXFILE = "index-v2"
+
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp, encoding: "UTF-8" }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/itu/bibliography.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Itu
+    module Bibliography
+      extend self
+
+      # @param refid [Relaton::Itu::Pubid, String] a document reference
+      # @return [Relaton::Itu::HitCollection]
+      def search(refid)
+        if refid.is_a? String
+          warn_incorrect_ref(refid)
+          refid = Pubid.parse refid
+        end
+        HitCollection.new(refid).tap(&:search)
+      end
+
+      # @param code [String] the ITU standard Code to look up
+      # @param year [String] the year the standard was published (optional)
+      # @param opts [Hash] options
+      # @return [Relaton::Bib::ItemData, nil]
+      def get(code, year = nil, opts = {})
+        warn_incorrect_ref(code)
+        refid = Pubid.parse code
+        refid.year ||= year
+
+        ret = itubib_get1(refid)
+        return nil if ret.nil?
+
+        ret = ret.to_most_recent_reference unless refid.year || opts[:keep_year]
+        ret = ret.to_all_parts if opts[:all_parts]
+        ret
+      end
+
+      private
+
+      def warn_incorrect_ref(ref)
+        if ref =~ /(ITU[\s-]T\s\w)\.(Suppl\.|Annex)\s?(\w?\d+)/
+          correct_ref = "#{$~[1]} #{$~[2]} #{$~[3]}"
+          Util.info "Incorrect reference: `#{ref}`, the reference should be: `#{correct_ref}`"
+        end
+      end
+
+      def fetch_ref_err(refid, missed_years)
+        Util.info "Not found.", key: refid.to_s
+        if missed_years.any?
+          plural = missed_years.size > 1 ? "s" : ""
+          Util.info "There was no match for `#{refid.year}` year, though there were matches " \
+                    "found for `#{missed_years.join('`, `')}` year#{plural}.", key: refid.to_s
+        end
+        nil
+      end
+
+      def search_filter(refid)
+        result = search(refid)
+        result.select do |i|
+          next true unless i.hit[:code]
+
+          pubid = Pubid.parse i.hit[:code]
+          refid.===(pubid, [:year])
+        end
+      end
+
+      def isobib_results_filter(result, refid)
+        missed_years = []
+        result.each do |r|
+          /\((?:\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
+          if !refid.year || refid.year == pyear
+            ret = r.item
+            return { ret: ret } if ret
+          end
+
+          missed_years << pyear
+        end
+        { years: missed_years }
+      end
+
+      def itubib_get1(refid)
+        result = search_filter(refid) || return
+        ret = isobib_results_filter(result, refid)
+        if ret[:ret]
+          Util.info "Found: `#{ret[:ret].docidentifier.first&.content}`", key: refid.to_s
+          ret[:ret]
+        else
+          fetch_ref_err(refid, ret[:years])
+        end
+      end
+    end
+  end
+end

data/lib/relaton/itu/data_fetcher.rb

@@ -0,0 +1,104 @@
+require "net/http"
+require "json"
+require "uri"
+require_relative "../itu"
+require_relative "data_parser_r"
+
+module Relaton
+  module Itu
+    class DataFetcher < Core::DataFetcher
+      SEARCH_URL = "https://www.itu.int/net4/ITU-T/search/GlobalSearch/RunSearch".freeze
+      ROWS = 100
+      MAX_EMPTY_PAGES = 3
+
+      def index
+        @index ||= Relaton::Index.find_or_create :itu, file: "index-v1.yaml"
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def fetch(_source = nil)
+        start = 0
+        empty_pages = 0
+        loop do
+          results = search_request(start)
+          if results.empty?
+            empty_pages += 1
+            break if empty_pages >= MAX_EMPTY_PAGES
+
+            start += ROWS
+            next
+          end
+
+          empty_pages = 0
+          results.each do |result|
+            bib = DataParserR.parse(result, @errors)
+            write_file(bib) if bib
+          rescue => e # rubocop:disable Style/RescueStandardError
+            Util.error "#{e.message}\n#{e.backtrace}"
+          end
+
+          start += ROWS
+        end
+        index.save
+        report_errors
+      end
+
+      # @param bib [Relaton::Itu::ItemData]
+      def write_file(bib) # rubocop:disable Metrics/AbcSize
+        id = bib.docidentifier.find(&:primary).content
+        file = output_file(id)
+        if @files.include? file
+          Util.warn "File #{file} exists."
+        else
+          @files << file
+        end
+        index.add_or_update id, file
+        File.write file, serialize(bib), encoding: "UTF-8"
+      end
+
+      def to_yaml(bib)
+        bib.to_yaml
+      end
+
+      def to_xml(bib)
+        bib.to_xml bibdata: true
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+
+      private
+
+      # @param start [Integer] pagination offset
+      # @return [Array<Hash>] search result items
+      def search_request(start)
+        payload = {
+          "Input" => "*", "Start" => start, "Rows" => ROWS,
+          "SortBy" => "DATE_NEW", "ExactPhrase" => false,
+          "CollectionName" => "ITU-R Publications",
+          "CollectionGroup" => "Publications", "Sector" => "r",
+          "Criterias" => [], "Topics" => "", "ClientData" => {},
+          "Language" => "en", "SearchType" => "All",
+        }
+
+        uri = URI(SEARCH_URL)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+
+        request = Net::HTTP::Post.new(uri)
+        request["Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8"
+        request["X-Requested-With"] = "XMLHttpRequest"
+        request["Referer"] = "https://www.itu.int/net4/itu-t/search/"
+        request.body = "json=#{URI.encode_www_form_component(payload.to_json)}"
+
+        response = http.request(request)
+        json = JSON.parse(response.body)
+        json["results"] || []
+      end
+    end
+  end
+end

data/lib/relaton/itu/data_parser_r.rb

@@ -0,0 +1,140 @@
+module Relaton
+  module Itu
+    module DataParserR
+      extend self
+
+      TYPE_MAP = {
+        "ITU-R Recommendations" => "recommendation",
+        "ITU-R Questions" => "question",
+        "ITU-R Reports" => "technical-report",
+        "Handbooks" => "handbook",
+        "ITU-R Resolutions" => "resolution",
+      }.freeze
+
+      #
+      # Parse ITU-R document from search API result.
+      #
+      # @param result [Hash] single search result from the API
+      #
+      # @return [Relaton::Itu::ItemData] bibliographic item
+      #
+      def parse(result, errors = {})
+        @errors = errors
+        doctype = fetch_doctype(result)
+        return unless doctype
+
+        Relaton::Itu::ItemData.new(
+          docidentifier: fetch_docid(result), title: fetch_title(result),
+          date: fetch_date(result), language: ["en"],
+          source: fetch_source(result), script: ["Latn"],
+          type: "standard", ext: Relaton::Itu::Ext.new(doctype: doctype, flavor: "itu"),
+        )
+      end
+
+      # @param result [Hash]
+      # @return [Array<Relaton::Bib::Docidentifier>]
+      def fetch_docid(result)
+        title = result["Title"].to_s
+        id = title.match(/^(ITU-R\s+\S+)/)&.captures&.first
+        return(@errors[:docid] &&= true; []) unless id
+
+        id = id.sub(/\s*\(.*/, "")
+        result_ids = [Docidentifier.new(type: "ITU", content: id, primary: true)]
+        @errors[:docid] &&= result_ids.empty?
+        result_ids
+      end
+
+      # @param result [Hash]
+      # @return [Array<Relaton::Bib::Title>]
+      def fetch_title(result)
+        title = result["Title"].to_s
+        content = title.sub(/^[^:]+:\s*/, "").strip
+        content = title unless content.length > 0
+        r = [Relaton::Bib::Title.new(type: "main", content: content, language: "en", script: "Latn")]
+        @errors[:title] &&= r.empty?
+        r
+      end
+
+      # @param result [Hash]
+      # @return [Array<Relaton::Bib::Date>]
+      def fetch_date(result)
+        prop = property(result, "Publication date")
+        unless prop
+          @errors[:date] &&= true
+          return []
+        end
+
+        date = parse_pub_date(prop)
+        unless date
+          @errors[:date] &&= true
+          return []
+        end
+
+        r = [Relaton::Bib::Date.new(type: "published", at: date)]
+        @errors[:date] &&= r.empty?
+        r
+      end
+
+      # @param result [Hash]
+      # @return [Array<Relaton::Bib::Uri>]
+      def fetch_source(result)
+        locations = result["Locations"]
+        unless locations.is_a?(Array)
+          @errors[:source] &&= true
+          return []
+        end
+
+        pdf = locations.find { |l| l["Type"] == "pdf" }
+        unless pdf && pdf["RawHref"]
+          @errors[:source] &&= true
+          return []
+        end
+
+        r = [Relaton::Bib::Uri.new(type: "pdf", content: pdf["RawHref"])]
+        @errors[:source] &&= r.empty?
+        r
+      end
+
+      # @param result [Hash]
+      # @return [Relaton::Itu::Doctype, nil]
+      def fetch_doctype(result)
+        type_value = property(result, "Type")
+        mapped = TYPE_MAP[type_value]
+        unless mapped
+          @errors[:doctype] &&= true
+          return
+        end
+
+        @errors[:doctype] &&= false
+        Doctype.new(content: mapped)
+      end
+
+      private
+
+      # Find a property value from the result's Properties array.
+      # @param result [Hash]
+      # @param name [String]
+      # @return [String, nil]
+      def property(result, name)
+        props = result["Properties"]
+        return unless props.is_a?(Array)
+
+        entry = props.find { |p| p["Title"] == name }
+        entry&.[]("Value")
+      end
+
+      # Parse publication date string like "January, 2024" or "2024".
+      # @param value [String]
+      # @return [String, nil]
+      def parse_pub_date(value)
+        case value
+        when /(\w+),?\s+(\d{4})/
+          month = Date::MONTHNAMES.index($1)
+          month ? "#{$2}-#{format('%02d', month)}" : $2
+        when /(\d{4})/
+          $1
+        end
+      end
+    end
+  end
+end

data/lib/relaton/itu/hit.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Itu
+    # Hit.
+    class Hit < Relaton::Core::Hit
+      attr_writer :item
+
+      # Parse page.
+      # @return [Relaton::Itu::ItemData]
+      def item
+        @item ||= Scraper.parse_page self, imp: gi_imp
+      end
+
+      private
+
+      def gi_imp
+        @gi_imp ||= /\.Imp\d/.match?(hit[:ref].to_s)
+      end
+    end
+  end
+end

data/lib/relaton/itu/hit_collection.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require_relative "hit"
+
+module Relaton
+  module Itu
+    # Page of hit collection.
+    class HitCollection < Relaton::Core::HitCollection
+      DOMAIN = "https://www.itu.int"
+      GH_ITU_R = "https://raw.githubusercontent.com/relaton/relaton-data-itu-r/refs/heads/v2/"
+
+      def search
+        case ref.to_ref
+        when /^(ITU-T|ITU-R\sRR)/
+          request_search
+        when /^ITU-R\s/
+          request_document
+        end
+      rescue Mechanize::ResponseCodeError, SocketError, Timeout::Error, Errno::ECONNRESET,
+              EOFError, Net::ProtocolError, OpenSSL::SSL::SSLError => e
+        raise Relaton::RequestError, "Could not access #{ref.to_ref}: #{e.message}"
+      end
+
+      def agent
+        @agent ||= Mechanize.new.tap { |agent| agent.user_agent_alias = "Mac Safari" }
+      end
+
+      private
+
+      def request_search
+        Util.info "Fetching from www.itu.int ...", key: ref.to_s
+        url = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/RunSearch"
+        data = { json: params.to_json }
+        resp = agent.post url, data
+        @array = hits JSON.parse(resp.body)
+      end
+
+      def request_document # rubocop:todo Metrics/MethodLength, Metrics/AbcSize
+        Util.info "Fetching from Relaton repository ...", key: ref.to_s
+        index = Relaton::Index.find_or_create :itu, url: "#{GH_ITU_R}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
+        row = index.search(ref.to_ref).max_by { |i| i[:id] }
+        return unless row
+
+        url = "#{GH_ITU_R}#{row[:file]}"
+        resp = agent.get url
+        return if resp.code == "404"
+
+        item = Item.from_yaml(resp.body).tap { |i| i.fetched = Date.today.to_s }
+        hit = Hit.new({ url: url, ref: ref }, self)
+        hit.item = item
+        @array = [hit]
+      end
+
+      # @return [String]
+      def group
+        @group ||= case ref.to_ref
+                   when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR}
+                     "Publications"
+                   when %r{^ITU-T} then "Recommendations"
+                   end
+      end
+
+      # @return [Hash]
+      def params # rubocop:disable Metrics/MethodLength
+        input = ref.dup
+        input.year = nil
+        {
+          "Input" => input.to_s,
+          "Start" => 0,
+          "Rows" => 20,
+          "SortBy" => "RELEVANCE",
+          "ExactPhrase" => false,
+          "CollectionName" => "General",
+          "CollectionGroup" => group,
+          "Sector" => ref.to_ref.match(/(?<=^ITU-)\w/).to_s.downcase,
+          "Criterias" => [{
+            "Name" => "Search in",
+            "Criterias" => [
+              {
+                "Selected" => false,
+                "Value" => "",
+                "Label" => "Name",
+                "Target" => "/name_s",
+                "TypeName" => "CHECKBOX",
+                "GetCriteriaType" => 0,
+              },
+              {
+                "Selected" => false,
+                "Value" => "",
+                "Label" => "Short description",
+                "Target" => "/short_description_s",
+                "TypeName" => "CHECKBOX",
+                "GetCriteriaType" => 0,
+              },
+              {
+                "Selected" => false,
+                "Value" => "",
+                "Label" => "File content",
+                "Target" => "/file",
+                "TypeName" => "CHECKBOX",
+                "GetCriteriaType" => 0,
+              },
+            ],
+            "ShowCheckbox" => true,
+            "Selected" => false,
+          }],
+          "Topics" => "",
+          "ClientData" => {},
+          "Language" => "en",
+          "SearchType" => "All",
+        }
+      end
+
+      # @param data [Hash]
+      # @return [Array<Relaton::Itu::Hit>]
+      def hits(data)
+        data["results"].map do |h|
+          code  = h["Media"]["Name"]
+          title = h["Title"]
+          url   = "#{DOMAIN}#{h['Redirection']}"
+          type  = h["Collection"]["Group"].downcase[0...-1]
+          Hit.new({ code: code, title: title, url: url, type: type }, self)
+        end
+      end
+    end
+  end
+end

data/lib/relaton/itu/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Itu
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/itu/model/approval_stage.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Itu
+    class ApprovalStage < Lutaml::Model::Serializable
+      attribute :process, :string, values: %w[tap aap]
+      attribute :content, :string, values: %w[determined in-force a lc ac lj aj na ar ri at sg c tap]
+
+      xml do
+        map_attribute "process", to: :process
+        map_content to: :content
+      end
+    end
+  end
+end

data/lib/relaton/itu/model/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Itu
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/itu/model/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Itu
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/itu/model/docidentifier.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Itu
+    class Docidentifier < Bib::Docidentifier
+      def remove_date!
+        self.content = content.gsub(/\s*\((?:\d{2}\/)?\d{4}\)/, "")
+      end
+    end
+  end
+end

data/lib/relaton/itu/model/doctype.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Itu
+    class Doctype < Bib::Doctype
+      TYPES = %w[
+        recommendation recommendation-supplement recommendation-amendment recommendation-corrigendum
+        recommendation-errata recommendation-annex focus-group implementers-guide technical-paper
+        technical-report joint-itu-iso-iec resolution service-publication handbook question contribution
+      ].freeze
+
+      attribute :content, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/itu/model/ext.rb

@@ -0,0 +1,47 @@
+require_relative "doctype"
+require_relative "question"
+require_relative "recommendation_status"
+require_relative "meeting"
+require_relative "meeting_date"
+require_relative "structured_identifier"
+
+module Relaton
+  module Itu
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :structuredidentifier, StructuredIdentifier
+      attribute :question, Question, collection: true
+      attribute :recommendationstatus, RecommendationStatus
+      attribute :ip_notice_received, :boolean
+      attribute :meeting, Meeting
+      attribute :meeting_place, :string
+      attribute :meeting_date, MeetingDate
+      attribute :intended_type, :string, values: %w[R C TD]
+      attribute :source, :string
+
+      xml do
+        map_element "question", to: :question
+        map_element "recommendationstatus", to: :recommendationstatus
+        map_element "ip-notice-received", to: :ip_notice_received
+        map_element "meeting", to: :meeting
+        map_element "meeting-place", to: :meeting_place
+        map_element "meeting-date", to: :meeting_date
+        map_element "intended-type", to: :intended_type
+        map_element "source", to: :source
+      end
+
+      key_value do
+        map_element "question", to: :question
+        map_element "recommendationstatus", to: :recommendationstatus
+        map_element "ip_notice_received", to: :ip_notice_received
+        map_element "meeting", to: :meeting
+        map_element "meeting_place", to: :meeting_place
+        map_element "meeting_date", to: :meeting_date
+        map_element "intended_type", to: :intended_type
+        map_element "source", to: :source
+      end
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-itu"]
+    end
+  end
+end

data/lib/relaton/itu/model/item.rb

@@ -0,0 +1,13 @@
+require_relative "docidentifier"
+require_relative "ext"
+
+module Relaton
+  module Itu
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/itu/model/meeting.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Itu
+    class Meeting < Lutaml::Model::Serializable
+      attribute :acronym, :string
+      attribute :content, :string
+
+      xml do
+        map_attribute "acronym", to: :acronym
+        map_content to: :content
+      end
+    end
+  end
+end

data/lib/relaton/itu/model/meeting_date.rb

@@ -0,0 +1,15 @@
+module Relaton
+  module Itu
+    class MeetingDate < Lutaml::Model::Serializable
+      attribute :from, :string
+      attribute :to, :string
+      attribute :at, :string
+
+      xml do
+        map_element "from", to: :from
+        map_element "to", to: :to
+        map_element "on", to: :at
+      end
+    end
+  end
+end

data/lib/relaton/itu/model/question.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Itu
+    class Question < Lutaml::Model::Serializable
+      attribute :identifier, :string
+      attribute :name, :string
+
+      xml do
+        map_element "identifier", to: :identifier
+        map_element "name", to: :name
+      end
+    end
+  end
+end