relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.1

data/lib/relaton/oiml.rb

@@ -0,0 +1,24 @@
+require "relaton/bib"
+require "relaton/index"
+require "pubid"
+require_relative "version"
+require_relative "oiml/util"
+require_relative "oiml/docidentifier"
+require_relative "oiml/item"
+require_relative "oiml/bibitem"
+require_relative "oiml/bibdata"
+require_relative "oiml/bibliography"
+
+module Relaton
+  module Oiml
+    INDEXFILE = "index-v2".freeze
+
+    class Error < StandardError; end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end

data/lib/relaton/omg/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Omg
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/omg/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Omg
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/omg/bibliography.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Omg
+    # OMG bibliography module
+    module Bibliography
+      extend self
+
+      # @param text [String] the OMG standard reference
+      # @return [Relaton::Omg::Item]
+      def search(text)
+        Scraper.scrape_page text
+      end
+
+      # @param code [String] the OMG standard reference
+      # @param year [String] the year the standard was published (optional)
+      # @param opts [Hash] options
+      # @return [Relaton::Omg::Item]
+      def get(code, _year = nil, _opts = {})
+        Util.info "Fetching from www.omg.org ...", key: code
+        result = search code
+        if result
+          Util.info "Found: `#{result.docidentifier.first.content}`", key: code
+        else
+          Util.info "Not found.", key: code
+        end
+        result
+      end
+    end
+  end
+end

data/lib/relaton/omg/ext.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Omg
+    class Ext < Bib::Ext
+      def get_schema_version = Relaton.schema_versions["relaton-model-omg"]
+    end
+  end
+end

data/lib/relaton/omg/item.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Omg
+    class Item < Bib::Item
+      model ItemData
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/omg/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Omg
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/omg/processor.rb

@@ -0,0 +1,34 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Omg
+    class Processor < Core::Processor
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_omg
+        @prefix = "OMG"
+        @defaultprefix = /^OMG /
+        @idtype = "OMG"
+      end
+
+      def get(code, date, opts)
+        require_relative "../omg"
+        Bibliography.get(code, date, opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../omg"
+        Bibitem.from_xml xml
+      end
+
+      def from_yaml(yaml)
+        require_relative "../omg"
+        Item.from_yaml yaml
+      end
+
+      def grammar_hash
+        require_relative "../omg"
+        @grammar_hash ||= Omg.grammar_hash
+      end
+    end
+  end
+end

data/lib/relaton/omg/scraper.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "mechanize"
+
+module Relaton
+  module Omg
+    class Scraper
+      URL_PATTERN = "https://www.omg.org/spec/"
+
+      def initialize(acronym, version = nil, spec = nil)
+        @acronym = acronym
+        @version = version
+        @spec = spec
+      end
+
+      def self.scrape_page(ref)
+        %r{^OMG (?<acronym>[^\s]+)(?:[\s/](?<version>[\d.]+(?:\sbeta(?:\s\d)?)?))?(?:[\s/](?<spec>\w+))?$} =~ ref
+        return unless acronym
+
+        scraper = new(acronym, version, spec)
+        doc = scraper.get_doc
+        return if doc.nil? || scraper.fetch_link.empty?
+
+        Omg::ItemData.new(**scraper.item)
+      end
+
+      def get_doc
+        @url = "#{URL_PATTERN}#{@acronym}/"
+        @url += @version.gsub(" ", "/") if @version
+        agent = Mechanize.new
+        agent.open_timeout = 10
+        @doc = agent.get(@url)
+      rescue Mechanize::ResponseCodeError => e
+        return if e.response_code == "404"
+
+        raise Relaton::RequestError, "Unable acces #{@url} (#{e.response_code})"
+      rescue Net::OpenTimeout
+        raise Relaton::RequestError, "Unable acces #{@url} (timeout)"
+      end
+
+      def item
+        {
+          fetched: ::Date.today.to_s,
+          docidentifier: fetch_docid,
+          title: fetch_title,
+          abstract: fetch_abstract,
+          version: fetch_version,
+          date: fetch_date,
+          status: fetch_status,
+          source: fetch_link,
+          relation: fetch_relation,
+          keyword: fetch_keyword,
+          license: fetch_license,
+        }
+      end
+
+      def fetch_title
+        content = @doc.at('//dt[.="Title:"]/following-sibling::dd').text
+        content += ": #{@spec}" if @spec
+        [Bib::Title.new(type: "main", content: content, language: "en", script: "Latn")]
+      end
+
+      def fetch_docid
+        id = ["OMG", @acronym]
+        id << doc_version if doc_version
+        id << @spec if @spec
+        [Bib::Docidentifier.new(content: id.join(" "), type: "OMG", primary: true)]
+      end
+
+      def fetch_abstract
+        content = @doc.at('//section[@id="document-metadata"]/div/div/p').text
+        [Bib::Abstract.new(content: content, language: "en", script: "Latn")]
+      end
+
+      def fetch_version
+        [Bib::Version.new(revision_date: pub_date, draft: doc_version)]
+      end
+
+      def doc_version
+        @doc_version ||= @doc.at('//dt[.="Version:"]/following-sibling::dd/p/span').text
+      end
+
+      def fetch_date
+        [Bib::Date.new(type: "published", at: pub_date.to_s)]
+      end
+
+      def pub_date
+        ::Date.parse @doc.at('//dt[.="Publication Date:"]/following-sibling::dd').text.strip
+      end
+
+      def fetch_status
+        status = @doc.at('//dt[.="Document Status:"]/following-sibling::dd')
+        stage = status.text.strip.match(/\w+/).to_s
+        Bib::Status.new(stage: Bib::Status::Stage.new(content: stage))
+      end
+
+      def fetch_link
+        return @links if @links
+
+        @links = []
+        if @spec
+          a = @doc.at("//a[@href='#{@url}/#{@spec}/PDF']")
+          @links << Bib::Uri.new(type: "src", content: a[:href]) if a
+        else
+          a = @doc.at('//dt[.="This Document:"]/following-sibling::dd/a')
+          @links << Bib::Uri.new(type: "src", content: a[:href]) if a
+          pdf = @doc.at('//a[@class="download-document"]')
+          @links << Bib::Uri.new(type: "pdf", content: pdf[:href]) if pdf
+        end
+        @links
+      end
+
+      def fetch_relation
+        v = @doc.xpath('//h2[.="History"]/following-sibling::section/div/table/tbody/tr')
+        v.reduce([]) do |mem, row|
+          ver = row.at("td").text
+          unless ver == doc_version
+            acronym = row.at("td[3]/a")[:href].split("/")[4]
+            id = ["OMG", acronym, ver].join(" ")
+            docid = Bib::Docidentifier.new(content: id, type: "OMG")
+            bibitem = Bib::ItemBase.new(formattedref: Bib::Formattedref.new(content: id), docidentifier: [docid])
+            mem << Bib::Relation.new(type: "obsoletes", bibitem: bibitem)
+          end
+          mem
+        end
+      end
+
+      def fetch_keyword
+        @doc.xpath('//dt[.="Categories:"]/following-sibling::dd/ul/li/a/em').map do |kw|
+          Bib::Keyword.new(vocab: Bib::LocalizedString.new(content: kw.text))
+        end
+      end
+
+      def fetch_license
+        @doc.xpath(
+          '//dt/span/a[contains(., "IPR Mode")]/../../following-sibling::dd/span',
+        ).map { |l| l.text.match(/[\w\s-]+/).to_s.strip }
+      end
+    end
+  end
+end

data/lib/relaton/omg/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Omg
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-omg".freeze
+    end
+  end
+end

data/lib/relaton/omg.rb

@@ -0,0 +1,24 @@
+require "nokogiri"
+require "relaton/bib"
+require_relative "version"
+require_relative "omg/util"
+require_relative "omg/ext"
+require_relative "omg/item_data"
+require_relative "omg/item"
+require_relative "omg/bibitem"
+require_relative "omg/bibdata"
+require_relative "omg/scraper"
+require_relative "omg/bibliography"
+
+module Relaton
+  module Omg
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/plateau/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Plateau
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/plateau/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Plateau
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/plateau/bibliography.rb

@@ -0,0 +1,24 @@
+module Relaton
+  module Plateau
+    module Bibliography
+      extend self
+
+      def search(code)
+        HitCollection.new(code).find
+      end
+
+      def get(code, _year = nil, _opts = {})
+        Util.info "Fetching ...", key: code
+        result = search(code).fetch_doc
+        if result
+          Util.info "Found `#{result.docidentifier.first.content}`", key: code
+          result
+        else
+          Util.warn "Not found.", key: code
+        end
+      rescue StandardError => e
+        raise Error, e.message
+      end
+    end
+  end
+end

data/lib/relaton/plateau/data_fetcher.rb

@@ -0,0 +1,153 @@
+require "json"
+require "relaton/core"
+require_relative "../plateau"
+require_relative "parser"
+require_relative "handbook_parser"
+require_relative "technical_report_parser"
+
+module Relaton
+  module Plateau
+    # Fetcher class to fetch data from the Plateau website
+    class DataFetcher < Core::DataFetcher
+      HANDBOOKS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/handbooks.json".freeze
+      TECHNICAL_REPORTS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/technical-reports.json".freeze
+
+      def index
+        @index ||= Relaton::Index.find_or_create :plateau, file: "#{INDEXFILE}.yaml"
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def fetch(source)
+        case source
+        when "plateau-handbooks" then extract_handbooks_data
+        when "plateau-technical-reports" then extract_technical_reports_data
+        else puts "Invalid source: #{source}"
+        end
+      end
+
+      # Create a GET request with custom headers to mimic a browser
+      def create_request(uri)
+        request = Net::HTTP::Get.new(uri)
+        request["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0"
+        request["Accept"] = "*/*"
+        request["Accept-Language"] = "en-US,en;q=0.5"
+        request["Accept-Encoding"] = "gzip, deflate, br, zstd"
+        request["Referer"] = "https://www.mlit.go.jp/plateau/libraries/"
+        request["purpose"] = "prefetch"
+        request["x-nextjs-data"] = "1"
+        request["Connection"] = "keep-alive"
+        request
+      end
+
+      # Handle different content encodings
+      def hadle_response(response)
+        if response["Content-Encoding"] == "gzip"
+          Zlib::GzipReader.new(StringIO.new(response.body)).read
+        elsif response["Content-Encoding"] == "deflate"
+          Zlib::Inflate.inflate(response.body)
+        else
+          response.body
+        end
+      end
+
+      # Fetch JSON data from a URL with custom headers
+      #
+      # @param [String] url The URL to fetch JSON data from
+      # @return [Hash] The parsed JSON data
+      def fetch_json_data(url)
+        uri = URI(url)
+
+        request = create_request(uri)
+
+        # Send the request and get the response
+        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request(request)
+        end
+
+        # Check if the response is successful
+        unless response.code.to_i == 200
+          Util.warn "Failed to fetch data: #{response.code} #{response.message}"
+          return {}
+        end
+
+        body = hadle_response(response)
+
+        # Parse the JSON response
+        JSON.parse(body)
+      rescue StandardError => e
+        # Handle any errors during the fetching process
+        Util.error "Error fetching JSON data from #{url}: #{e.message}"
+        {}
+      end
+
+      #
+      # Extract data for handbooks
+      #
+      def extract_handbooks_data
+        data = fetch_json_data(HANDBOOKS_URL)
+        Util.info "Extracting handbooks data..."
+        data["pageProps"]["handbooks"]["nodes"].each do |entry|
+          handbook = entry["handbook"]
+          doctype = entry["slug"].match("-") ? "annex" : "handbook"
+
+          handbook["versions"].each do |version|
+            item = HandbookParser.new(version: version, entry: entry, doctype: doctype, errors: @errors).parse
+            save_document(item)
+          end
+        end
+        index.save
+        report_errors
+      end
+
+      #
+      # Extract data for technical reports
+      #
+      def extract_technical_reports_data
+        data = fetch_json_data(TECHNICAL_REPORTS_URL)
+        Util.info "Extracting technical reports data..."
+        data["pageProps"]["nodes"].map do |entry|
+          save_document(TechnicalReportParser.new(entry, @errors).parse)
+        end
+        index.save
+        report_errors
+      end
+
+      def save_document(item)
+        id = item.docidentifier.first.content
+        file = file_name id
+        if @files.include?(file)
+          Util.warn "File #{file} already exists, skipping.", key: id
+        else
+          File.write(file, serialize(item))
+          @files << file
+          index.add_or_update id, file
+        end
+      end
+
+      def file_name(id)
+        name = id.gsub(/\s+/, "-").gsub(/[^\w-]+/, "").downcase
+        if id.match?(/民間活用編/)
+          name += "-private"
+        elsif id.match?(/公共活用編/)
+          name += "-public"
+        end
+        File.join(@output, "#{name}.#{@ext}")
+      end
+
+      def to_yaml(bib)
+        Item.to_yaml(bib)
+      end
+
+      def to_xml(bib)
+        Item.to_xml(bib, bibdata: true)
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+    end
+  end
+end

data/lib/relaton/plateau/doctype.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Plateau
+    class Doctype < Bib::Doctype
+      attribute :content, :string, values: %w[handbook technical-report annex]
+    end
+  end
+end

data/lib/relaton/plateau/ext.rb

@@ -0,0 +1,23 @@
+require_relative "doctype"
+
+module Relaton
+  module Plateau
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :stagename, Iso::Stagename
+      attribute :filesize, :integer
+
+      xml do
+        map_element "stagename", to: :stagename
+        map_element "filesize", to: :filesize
+      end
+
+      key_value do
+        map_element "stagename", to: :stagename
+        map_element "filesize", to: :filesize
+      end
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-plateau"]
+    end
+  end
+end

data/lib/relaton/plateau/handbook_parser.rb

@@ -0,0 +1,95 @@
+# encoding: UTF-8
+
+module Relaton
+  module Plateau
+    class HandbookParser < Parser
+      def initialize(version:, entry:, doctype:, errors: {})
+        @version = version
+        @entry = entry
+        super(entry["handbook"], errors)
+        @doctype = doctype
+      end
+
+      private
+
+      def edition
+        @edition ||= @version["title"].split.first.match(/[\d.]+/).to_s
+      end
+
+      def slug_number
+        @slug_number ||= @entry["slug"]&.to_s&.split("_")&.first
+      end
+
+      def parse_docnumber
+        @errors[:hb_docnumber] &&= @entry["slug"].nil? || @entry["slug"].to_s.empty?
+        ["Handbook ##{slug_number}", edition].compact.join(" ")
+      end
+
+      def parse_abstract
+        unless @item["description"]
+          @errors[:hb_abstract] &&= true
+          return []
+        end
+
+        result = @item["description"].split("<br />").filter_map do |part|
+          text = part.strip
+          next if text.empty?
+
+          lang, script = detect_lang(text)
+          create_abstract(text, lang, script)
+        end
+        @errors[:hb_abstract] &&= result.empty?
+        result
+      end
+
+      def parse_edition
+        if edition.nil? || edition.empty?
+          @errors[:hb_edition] &&= true
+          return
+        end
+
+        number = edition.match(/\d\.\d/)[0]
+        result = Bib::Edition.new(content: edition, number: number)
+        @errors[:hb_edition] &&= result.nil?
+        result
+      end
+
+      def parse_date
+        if @version["date"].nil? || @version["date"].empty?
+          @errors[:hb_date] &&= true
+          return super
+        end
+
+        result = super << create_date(@version["date"].gsub(".", "-"))
+        @errors[:hb_date] &&= result.empty?
+        result
+      end
+
+      def parse_source
+        result = %w[pdf html].map do |type|
+          next unless @version[type]
+
+          create_link(@version[type], type)
+        end.compact
+        @errors[:hb_source] &&= result.empty?
+        result
+      end
+
+      def parse_ext
+        strid = Bib::StructuredIdentifier.new(
+          type: "Handbook", agency: ["PLATEAU"], docnumber: slug_number, edition: edition
+        )
+        Ext.new(
+          doctype: Doctype.new(content: @doctype),
+          flavor: "plateau",
+          structuredidentifier: [strid],
+          filesize: filesize
+        )
+      end
+      def filesize
+        @errors[:hb_filesize] &&= @version["filesize"].nil?
+        @version["filesize"].to_i
+      end
+    end
+  end
+end

data/lib/relaton/plateau/hit.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Plateau
+    class Hit < Relaton::Core::Hit
+      def item
+        @item ||= begin
+          url = "#{HitCollection::ENDPOINT}#{hit[:file]}"
+          resp = Net::HTTP.get_response(URI(url))
+          Item.from_yaml(resp.body)
+        end
+      end
+    end
+  end
+end