relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/plateau/hit_collection.rb

@@ -0,0 +1,59 @@
+require_relative "hit"
+
+module Relaton
+  module Plateau
+    class HitCollection < Relaton::Core::HitCollection
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-plateau/v2/"
+
+      def find
+        @array = index.search do |row|
+          if all_editions?
+            row[:id].sub(/ \d+\.\d+$/, "") == @ref
+          else
+            row[:id] == @ref
+          end
+        end.map { |row| Hit.new(row, self) }
+        self
+      end
+
+      def fetch_doc
+        return unless any?
+
+        all_editions? ? to_all_editions : first.item
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :plateau, url: "#{ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
+        )
+      end
+
+      private
+
+      def all_editions?
+        @ref.match?(/ #\d+$/)
+      end
+
+      def to_all_editions
+        return first.item if size < 2
+
+        bibitem = first.item
+        relations = map do |h|
+          Bib::Relation.new(type: "hasEdition", bibitem: h.item)
+        end
+        docid = bibitem.docidentifier.map do |d|
+          Bib::Docidentifier.new(
+            content: d.content.sub(/ \d+\.\d+$/, ""), type: d.type, primary: d.primary
+          )
+        end
+        ItemData.new(
+          docidentifier: docid,
+          docnumber: bibitem.docnumber.sub(/ \d+\.\d+$/, ""),
+          title: bibitem.title,
+          contributor: bibitem.contributor,
+          relation: relations
+        )
+      end
+    end
+  end
+end

data/lib/relaton/plateau/item.rb

@@ -0,0 +1,12 @@
+require_relative "ext"
+require_relative "item_data"
+
+module Relaton
+  module Plateau
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/plateau/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Plateau
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/plateau/parser.rb

@@ -0,0 +1,101 @@
+module Relaton
+  module Plateau
+    # Base class for Plateau parsers
+    class Parser
+      ATTRIS = %i[docidentifier docnumber title abstract depiction edition type
+                  date source contributor keyword ext].freeze
+
+      def initialize(item, errors = {})
+        @item = item
+        @errors = errors
+      end
+
+      def parse
+        args = ATTRIS.each_with_object({}) do |attr, hash|
+          hash[attr] = send("parse_#{attr}")
+        end
+        ItemData.new(**args)
+      end
+
+      private
+
+      def parse_docidentifier
+        result = [create_docid("PLATEAU #{parse_docnumber}")]
+        @errors[:parse_docidentifier] &&= result.empty?
+        result
+      end
+
+      def parse_docnumber; end
+
+      def create_docid(id)
+        Bib::Docidentifier.new(type: "PLATEAU", content: id, primary: true)
+      end
+
+      def create_abstract(content, lang = "ja", script = "Jpan")
+        Bib::Abstract.new(content: content, language: lang, script: script)
+      end
+
+      def detect_lang(text)
+        if text&.match?(/[\p{Han}\p{Katakana}\p{Hiragana}]/)
+          ["ja", "Jpan"]
+        else
+          ["en", "Latn"]
+        end
+      end
+
+      def parse_title
+        lang, script = detect_lang(@item["title"])
+        result = [create_title(@item["title"], lang, script)]
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      def create_title(title, lang, script)
+        Bib::Title.new(type: "main", content: title, language: lang, script: script)
+      end
+
+      def parse_abstract; [] end
+
+      def parse_depiction
+        image_ext = @item["thumbnail"]["mediaItemUrl"].split(".").last
+        mimetype = "image/"
+        mimetype += image_ext == "jpg" ? "jpeg" : image_ext
+        src = "https://www.mlit.go.jp/#{@item["thumbnail"]["mediaItemUrl"]}"
+        image = Bib::Image.new(src: src, mimetype: mimetype)
+        result = Bib::Depiction.new(scope: "cover", image: [image])
+        @errors[:parse_depiction] &&= result.nil?
+        [result]
+      end
+
+      def parse_edition; raise "Not implemented" end
+      def parse_type; "standard" end
+      def parse_date; [] end
+      def parse_source; [] end
+
+      def parse_contributor
+        name = [
+          Bib::TypedLocalizedString.new(content: "国土交通省", language: "ja", script: "Jpan"),
+          Bib::TypedLocalizedString.new(
+            content: "Japanese Ministry of Land, Infrastructure, Transport and Tourism",
+            language: "en", script: "Latn"
+          ),
+        ]
+        org = Bib::Organization.new(name: name, abbreviation: Bib::LocalizedString.new(content: "MLIT"))
+        result = [Bib::Contributor.new(organization: org, role: [Bib::Contributor::Role.new(type: "publisher")])]
+        @errors[:parse_contributor] &&= result.empty?
+        result
+      end
+
+      def create_date(date, type = "published")
+        Bib::Date.new(type: type, at: date)
+      end
+
+      def create_link(url, type)
+        Bib::Uri.new(type: type, content: url)
+      end
+
+      def parse_keyword; [] end
+      def parse_ext; raise "Not implemented" end
+    end
+  end
+end

data/lib/relaton/plateau/processor.rb

@@ -0,0 +1,49 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Plateau
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_plateau
+        @prefix = "PLATEAU"
+        @defaultprefix = /^PLATEAU\s/
+        @idtype = "PLATEAU"
+        @datasets = %w[plateau-handbooks plateau-technical-reports]
+      end
+
+      def get(code, date, opts)
+        require_relative "../plateau"
+        Bibliography.get(code, date, opts)
+      end
+
+      def fetch_data(source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(source, **opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../plateau"
+        Item.from_xml xml
+      end
+
+      def from_yaml(yaml)
+        require_relative "../plateau"
+        Item.from_yaml yaml
+      end
+
+      def grammar_hash
+        require_relative "../plateau"
+        @grammar_hash ||= ::Relaton::Plateau.grammar_hash
+      end
+
+      def threads = 3
+
+      def remove_index_file
+        require_relative "../plateau"
+        Relaton::Index.find_or_create(:plateau, url: true, file: "#{INDEXFILE}.yaml").remove_file
+      end
+    end
+  end
+end

data/lib/relaton/plateau/technical_report_parser.rb

@@ -0,0 +1,91 @@
+module Relaton
+  module Plateau
+    class TechnicalReportParser < Parser
+      def initialize(entry, errors = {})
+        @entry = entry
+        super(entry["technicalReport"], errors)
+      end
+
+      private
+
+      def parse_docnumber
+        @errors[:tr_docnumber] &&= @entry["slug"].nil? || @entry["slug"].to_s.empty?
+        "Technical Report ##{@entry["slug"]} #{edition_number}"
+      end
+
+      def parse_abstract
+        if @item["subtitle"].nil? || @item["subtitle"].empty?
+          @errors[:tr_abstract] &&= true
+          return super
+        end
+
+        result = super << create_abstract(@item["subtitle"])
+        @errors[:tr_abstract] &&= result.empty?
+        result
+      end
+
+      def parse_edition
+        result = Bib::Edition.new(content: edition_number, number: edition_number)
+        result
+      end
+
+      def edition_number = "1.0"
+
+      def parse_date
+        if @entry["date"].nil? || @entry["date"].empty?
+          @errors[:tr_date] &&= true
+          return super
+        end
+
+        date_str = @entry["date"].sub(/T.*/, "")
+        result = super << create_date(date_str)
+        @errors[:tr_date] &&= result.empty?
+        result
+      end
+
+      def parse_source
+        if @item["pdf"].nil? || @item["pdf"].empty?
+          @errors[:tr_source] &&= true
+          return super
+        end
+
+        result = super << create_link(@item["pdf"], "pdf")
+        @errors[:tr_source] &&= result.empty?
+        result
+      end
+
+      def parse_keyword
+        result = @entry["globalTags"]["nodes"].map do |tag|
+          Bib::Keyword.new(vocab: Bib::LocalizedString.new(content: tag["name"]))
+        end
+        @errors[:tr_keyword] &&= result.empty?
+        result
+      end
+
+      def parse_ext
+        strid = Bib::StructuredIdentifier.new(
+          type: "Technical Report", klass: parse_subdoctype,
+          agency: ["PLATEAU"], docnumber: @entry["slug"]
+        )
+        Ext.new(
+          doctype: Doctype.new(content: "technical-report"),
+          subdoctype: parse_subdoctype,
+          flavor: "plateau",
+          structuredidentifier: [strid],
+          filesize: filesize
+        )
+      end
+
+      def filesize
+        @errors[:tr_filesize] &&= @item["filesize"].nil?
+        @item["filesize"].to_i
+      end
+
+      def parse_subdoctype
+        result = @entry["technicalReportCategories"]["nodes"].dig(0, "name")
+        @errors[:tr_subdoctype] &&= result.nil?
+        result
+      end
+    end
+  end
+end

data/lib/relaton/plateau/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Plateau
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-plateau".freeze
+    end
+  end
+end

data/lib/relaton/plateau.rb

@@ -0,0 +1,28 @@
+require "net/http"
+require "uri"
+require "relaton/index"
+require "relaton/iso"
+require_relative "version"
+require_relative "plateau/util"
+# require_relative "plateau/docidentifier"
+# require_relative "plateau/document_type"
+require_relative "plateau/item"
+require_relative "plateau/bibitem"
+require_relative "plateau/bibdata"
+require_relative "plateau/hit_collection"
+require_relative "plateau/bibliography"
+require_relative "plateau/processor"
+# require_relative "plateau/xml_parser"
+# require_relative "plateau/hash_converter"
+
+module Relaton
+  module Plateau
+    INDEXFILE = "index-v1"
+
+    class Error < StandardError; end
+
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/un/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Un
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/un/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Un
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/un/bibliography.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Un
+    # Class methods for search UN standards.
+    class Bibliography
+      class << self
+        # @param text [String]
+        # @return [Relaton::Un::HitCollection]
+        def search(text)
+          HitCollection.search text
+        rescue Faraday::ConnectionFailed, Faraday::TimeoutError,
+               Faraday::SSLError => e
+          raise Relaton::RequestError,
+                "Could not access #{HitCollection::API_BASE}: #{e.message}"
+        end
+
+        # @param ref [String] document reference
+        # @param year [String, NilClass]
+        # @param opts [Hash] options
+        # @return [Relaton::Bib::ItemData]
+        def get(ref, _year = nil, _opts = {})
+          Util.info "Fetching from documents.un.org ...", key: ref
+          /^(?:UN\s)?(?<code>.*)/ =~ ref
+          result = isobib_search_filter(code)
+          if result
+            Util.info "Found: `#{result.item.docidentifier[0].content}`", key: ref
+            result.item
+          else
+            Util.info "Not found.", key: ref
+            nil
+          end
+        end
+
+        private
+
+        # Search for hits.
+        #
+        # @param code [String] reference without correction
+        # @return [Relaton::Un::Hit, nil]
+        def isobib_search_filter(code)
+          result = search(code)
+          result.select { |i| i.hit["symbols"]&.compact&.include?(code) }.first
+        end
+      end
+    end
+  end
+end

data/lib/relaton/un/doctype.rb

@@ -0,0 +1,10 @@
+module Relaton
+  module Un
+    class Doctype < Bib::Doctype
+      attribute :content, :string, values: %w[
+        recommendation plenary addendum communication corrigendum
+        reissue agenda budgetary sec-gen-notes expert-report resolution
+      ]
+    end
+  end
+end

data/lib/relaton/un/ext.rb

@@ -0,0 +1,30 @@
+require_relative "doctype"
+require_relative "session"
+
+module Relaton
+  module Un
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :submissionlanguage, :string, collection: true
+      attribute :distribution, :string, values: %w[general limited restricted provisional]
+      attribute :session, Session
+      attribute :job_number, :string
+
+      xml do
+        map_element "submissionlanguage", to: :submissionlanguage
+        map_element "distribution", to: :distribution
+        map_element "session", to: :session
+        map_element "job_number", to: :job_number
+      end
+
+      key_value do
+        map "submissionlanguage", to: :submissionlanguage
+        map "distribution", to: :distribution
+        map "session", to: :session
+        map "job_number", to: :job_number
+      end
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-un"]
+    end
+  end
+end

data/lib/relaton/un/hit.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Un
+    class Hit < Core::Hit
+      # rubocop:disable Layout/LineLength
+      BODY = {
+        "A" => "General Assembly",
+        "E" => "Economic and Social Council",
+        "S" => "Security Council",
+        "T" => "Trusteeship Council",
+        "ACC" => "Administrative Committee on Coordination",
+        "AT" => "United Nations Administrative Tribunal",
+        "CAT" => "Committee against Torture",
+        "CCPR" => "Human Rights Committee",
+        "CD" => "Conference on Disarmament",
+        "CEDAW" => "Committee on the Elimination of All Forms of Discrimination against Women",
+        "CERD" => "Committee on the Elimination of Racial Discrimination",
+        "CRC" => "Committee on the Rights of the Child",
+        "DC" => "Disarmament Commission",
+        "DP" => "United Nations Development Programme",
+        "HS" => "United Nations Centre for Human Settlements (HABITAT)",
+        "TD" => "United Nations Conference on Trade and Development",
+        "UNEP" => "United Nations Environment Programme",
+        "TRADE" => "Committee on Trade",
+        "CEFACT" => "Centre for Trade Facilitation and Electronic Business",
+        "C.1" => "Disarmament and International Security Committee",
+        "C.2" => "Economic and Financial Committee",
+        "C.3" => "Social, Humanitarian & Cultural Issues",
+        "C.4" => "Special Political and Decolonization Committee",
+        "C.5" => "Administrative and Budgetary Committee",
+        "C.6" => "Sixth Committee (Legal)",
+        "PC" => "Preparatory Committee",
+        "AEC" => "Atomic Energy Commission",
+        "AGRI" => "Committee on Agriculture",
+        "AMCEN" => "African Ministerial Conference on the Environment",
+        "AMCOW" => "African Ministers' Council on Water",
+        "ECA" => "Economic Commission for Africa",
+        "ESCAP" => "Economic and Social Commission for Asia and Pacific",
+        "ECE" => "Economic Commission for Europe",
+        "ECWA" => "Economic Commission for Western Asia",
+        "UNFF" => "United Nations Forum on Forests",
+        "ENERGY" => "Committee on Sustainable Energy",
+        "FAO" => "Food and Agriculture Organization",
+        "UNCTAD" => "United Nations Conference on Trade and Development",
+      }.freeze
+      # rubocop:enable Layout/LineLength
+
+      # Parse page.
+      # @return [Relaton::Un::Item]
+      def item
+        @item ||= Parser.new(hit).parse
+      end
+    end
+  end
+end

data/lib/relaton/un/hit_collection.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+require "securerandom"
+
+module Relaton
+  module Un
+    class HitCollection < Core::HitCollection
+      API_BASE = "https://documents.un.org/api"
+
+      # Factory method — performs the API search and populates hits.
+      # @param text [String] symbol to search for
+      # @return [HitCollection]
+      def self.search(text)
+        hc = new(text)
+        hc.send(:fetch_hits)
+        hc
+      end
+
+      private
+
+      def fetch_hits
+        response = search_api(@ref)
+        response&.each { |data| @array << Hit.new(data, self) }
+      end
+
+      def search_api(text) # rubocop:disable Metrics/MethodLength
+        resp = connection.post("search?l=en&rid=#{SecureRandom.uuid}") do |req|
+          req.headers["Content-Type"] = "application/json"
+          req.body = JSON.generate(search_body(text))
+        end
+        return unless resp.success?
+
+        body = JSON.parse(resp.body)
+        return unless body["status"] == 1
+
+        body.dig("body", "data")
+      end
+
+      def search_body(text) # rubocop:disable Metrics/MethodLength
+        {
+          symbol: text, jobNumber: "", publicationDate: "* TO *",
+          releaseDate: "* TO *", title: "", subject: "", session: "",
+          agenda: "", truncation: false,
+          fullTextSearch: { language: "en", searchText: "" },
+          sortOptions: { sortField: "Sort by relevance" },
+          pagination: { currentPage: 1, itemsPerPage: 20 },
+          screenLanguage: "en", tcodes: [],
+        }
+      end
+
+      def connection
+        @connection ||= Faraday.new(url: API_BASE) do |f|
+          f.headers["Authorization"] = "Access #{TokenGenerator.generate}"
+          f.headers["Cache-Control"] = "public, max-age=0"
+        end
+      end
+    end
+  end
+end

data/lib/relaton/un/item.rb

@@ -0,0 +1,11 @@
+require_relative "ext"
+
+module Relaton
+  module Un
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/un/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Un
+    class ItemData < Bib::ItemData
+    end
+  end
+end