relaton-plateau 1.19.0

data/grammar/relaton-plateau-compile.rng

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0">
+  <include href="basicdoc.rng"/>
+  <include href="relaton-plateau.rng"/>
+  <start>
+    <choice>
+      <ref name="bibitem"/>
+      <ref name="bibdata"/>
+    </choice>
+  </start>
+</grammar>

data/grammar/relaton-plateau.rng

@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<grammar xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <include href="biblio-standoc.rng">
+    <define name="BibDataExtensionType">
+      <optional>
+        <attribute name="schema-version"/>
+      </optional>
+      <ref name="doctype"/>
+      <optional>
+        <ref name="docsubtype"/>
+      </optional>
+      <optional>
+        <ref name="editorialgroup"/>
+      </optional>
+      <zeroOrMore>
+        <ref name="ics"/>
+      </zeroOrMore>
+      <ref name="structuredidentifier"/>
+      <optional>
+        <ref name="stagename"/>
+      </optional>
+      <optional>
+        <ref name="cover"/>
+      </optional>
+      <optional>
+        <ref name="filesize"/>
+      </optional>
+    </define>
+    <define name="DocumentType">
+      <choice>
+        <value>handbook</value>
+        <value>technical-report</value>
+        <value>annex</value>
+      </choice>
+    </define>
+    <define name="editorialgroup">
+      <element name="editorialgroup">
+        <ref name="ISOProjectGroup"/>
+      </element>
+    </define>
+  </include>
+  <define name="ISOProjectGroup">
+    <zeroOrMore>
+      <ref name="agency"/>
+    </zeroOrMore>
+    <oneOrMore>
+      <ref name="technical-committee"/>
+    </oneOrMore>
+    <zeroOrMore>
+      <ref name="subcommittee"/>
+    </zeroOrMore>
+    <zeroOrMore>
+      <ref name="workgroup"/>
+    </zeroOrMore>
+    <optional>
+      <ref name="secretariat"/>
+    </optional>
+  </define>
+  <define name="agency">
+    <element name="agency">
+      <text/>
+    </element>
+  </define>
+  <define name="documentnumber">
+    <element name="project-number">
+      <optional>
+        <attribute name="part">
+          <data type="int"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="subpart">
+          <data type="int"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="amendment">
+          <data type="int"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="corrigendum">
+          <data type="int"/>
+        </attribute>
+      </optional>
+      <optional>
+        <attribute name="origyr">
+          <ref name="ISO8601Date"/>
+        </attribute>
+      </optional>
+      <text/>
+    </element>
+  </define>
+  <define name="subcommittee">
+    <element name="subcommittee">
+      <ref name="IsoWorkgroup"/>
+    </element>
+  </define>
+  <define name="workgroup">
+    <element name="workgroup">
+      <ref name="IsoWorkgroup"/>
+    </element>
+  </define>
+  <define name="secretariat">
+    <element name="secretariat">
+      <text/>
+    </element>
+  </define>
+  <define name="stagename">
+    <element name="stagename">
+      <optional>
+        <attribute name="abbreviation"/>
+      </optional>
+      <text/>
+    </element>
+  </define>
+  <define name="cover">
+    <element name="cover">
+      <ref name="image-no-id"/>
+    </element>
+  </define>
+  <define name="filesize">
+    <element name="filesize">
+      <data type="int"/>
+    </element>
+  </define>
+</grammar>

data/lib/relaton/plateau/bibitem.rb

@@ -0,0 +1,82 @@
+require_relative "cover"
+require_relative "stagename"
+
+module Relaton
+  module Plateau
+    class BibItem < RelatonBib::BibliographicItem
+      # @return [Relaton::Plateau::Cover]
+      attr_reader :cover
+
+      # @return [Relaton::Plateau::Stagename]
+      attr_reader :stagename
+
+      # @return [Integer]
+      attr_reader :filesize
+
+      def initialize(**args)
+        @cover = args.delete(:cover)
+        @filesize = args.delete(:filesize)
+        @stagename = args.delete(:stagename)
+        super(**args)
+      end
+
+      #
+      # Fetch flavor schema version
+      #
+      # @return [String] schema version
+      #
+      def ext_schema
+        @ext_schema ||= schema_versions["relaton-model-plateau"]
+      end
+
+      # @param opts [Hash]
+      # @option opts [Nokogiri::XML::Builder] :builder XML builder
+      # @option opts [Boolean] bibdata
+      # @option opts [Symbol, nil] :date_format (:short), :full
+      # @option opts [String] :lang language
+      def to_xml(**opts)
+        super do |builder|
+          if opts[:bibdata] && has_ext_data?
+            ext = builder.ext do |b|
+              doctype&.to_xml b
+              b.subdoctype subdoctype if subdoctype
+              editorialgroup&.to_xml b
+              ics.each { |i| b.ics i }
+              structuredidentifier&.to_xml b
+              stagename&.to_xml b
+              cover.to_xml b
+              b.filesize filesize
+            end
+            ext["schema-version"] = ext_schema if !opts[:embedded] && respond_to?(:ext_schema) && ext_schema
+          end
+        end
+      end
+
+      def to_hash
+        hash = super
+        return hash unless has_ext_data?
+
+        hash["ext"] ||= {}
+        hash["ext"]["stagename"] = stagename.to_hash if stagename
+        hash["ext"]["cover"] = cover.to_hash if cover
+        hash["ext"]["filesize"] = filesize if filesize
+        hash
+      end
+
+      def to_asciibib(prefix = "")
+        pref = prefix.empty? ? "" : "#{prefix}."
+        output = super
+        output += stagename.to_asciibib prefix if stagename
+        output += cover.to_asciibib prefix if cover
+        output += "#{pref}filesize:: #{filesize}\n" if filesize
+        output
+      end
+
+      private
+
+      def has_ext_data?
+        doctype || subdoctype || editorialgroup || ics&.any? || structuredidentifier || stagename || cover || filesize
+      end
+    end
+  end
+end

data/lib/relaton/plateau/bibliography.rb

@@ -0,0 +1,44 @@
+module Relaton
+  module Plateau
+    module Bibliography
+      extend self
+
+      INDEXFILE = "index-v1"
+      GHURL = "https://raw.githubusercontent.com/relaton/relaton-data-plateau/main/"
+
+      def index
+        Relaton::Index.find_or_create :plateau, url: "#{GHURL}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
+      end
+
+      def get(code, year = nil, opts = {})
+        Util.info "Fetching ...", key: code
+        bib = search(code)
+        if bib
+          Util.info "Found `#{bib.docidentifier.first.id}`", key: code
+          bib
+        else
+          Util.warn "Not found.", key: code
+        end
+      rescue StandardError => e
+        raise RelatonBib::RequestError, e.message
+      end
+
+      def search(code)
+        rows = index.search(code)
+        return unless rows.any?
+
+        row = rows.sort_by { |r| r[:id] }.last
+        fetch_doc code, **row
+      end
+
+      def fetch_doc(code, id:, file:)
+        resp = Net::HTTP.get_response URI("#{GHURL}#{file}")
+        return unless resp.is_a? Net::HTTPSuccess
+
+        hash = YAML.load(resp.body)
+        args = HashConverter.hash_to_bib hash
+        BibItem.new(**args)
+      end
+    end
+  end
+end

data/lib/relaton/plateau/cover.rb

@@ -0,0 +1,32 @@
+module Relaton
+  module Plateau
+    class Cover
+      # @return [RelatonBib::Image]
+      attr_reader :image
+
+      #
+      # Initialize the Cover object
+      #
+      # @param [RelatonBib::Image] image image object
+      #
+      def initialize(image)
+        @image = image
+      end
+
+      def to_xml(builder)
+        builder.cover do |b|
+          image.to_xml b
+        end
+      end
+
+      def to_hash
+        image.to_hash
+      end
+
+      def to_asciibib(prefix = "")
+        pref = prefix.empty? ? "cover" : "#{prefix}.cover"
+        image.to_asciibib pref
+      end
+    end
+  end
+end

data/lib/relaton/plateau/document_type.rb

@@ -0,0 +1,18 @@
+module Relaton
+  module Plateau
+    class DocumentType < RelatonBib::DocumentType
+      DOCTYPES = %w[handbook technical-report annex].freeze
+
+      def initialize(type:, abbreviation: nil)
+        check_type type
+        super
+      end
+
+      def check_type(type)
+        return if DOCTYPES.include? type
+
+        Util.warn "invalid doctype: `#{type}`"
+      end
+    end
+  end
+end

data/lib/relaton/plateau/fetcher.rb

@@ -0,0 +1,219 @@
+require "json"
+require_relative "parser"
+require_relative "handbook_parser"
+require_relative "technical_report_parser"
+
+module Relaton
+  module Plateau
+    # Fetcher class to fetch data from the Plateau website
+    class Fetcher
+      HANDBOOKS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/handbooks.json".freeze
+      TECHNICAL_REPORTS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/technical-reports.json".freeze
+
+      def initialize(output, format)
+        @output = output
+        @format = format
+        @ext = format.sub(/^bib/, "")
+        @files = []
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create :plateau, file: "index-v1.yaml"
+      end
+
+      def self.fetch(source, output: "data", format: "yaml")
+        t1 = Time.now
+        puts "Started at: #{t1}"
+        FileUtils.mkdir_p output
+
+        if source == "plateau-handbooks"
+          new(output, format).extract_handbooks_data
+        elsif source == "plateau-technical-reports"
+          new(output, format).extract_technical_reports_data
+        else
+          puts "Invalid source: #{source}"
+        end
+
+        t2 = Time.now
+        puts "Stopped at: #{t2}"
+        puts "Done in: #{(t2 - t1).round} sec."
+      end
+
+      # def fetch_handbooks
+      #   data = fetch_json_data(HANDBOOKS_URL)
+      #   extracted_data = extract_handbooks_data(data)
+      #   save_to_yaml(extracted_data, "handbooks.yaml")
+      # end
+
+      # def fetch_technical_reports
+      #   data = fetch_json_data(TECHNICAL_REPORTS_URL)
+      #   extracted_data = extract_technical_reports_data(data)
+      #   save_to_yaml(extracted_data, "technical_reports.yaml")
+      # end
+
+      # Create a GET request with custom headers to mimic a browser
+      def create_request(uri)
+        request = Net::HTTP::Get.new(uri)
+        request["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0"
+        request["Accept"] = "*/*"
+        request["Accept-Language"] = "en-US,en;q=0.5"
+        request["Accept-Encoding"] = "gzip, deflate, br, zstd"
+        request["Referer"] = "https://www.mlit.go.jp/plateau/libraries/"
+        request["purpose"] = "prefetch"
+        request["x-nextjs-data"] = "1"
+        request["Connection"] = "keep-alive"
+        request
+      end
+
+      # Handle different content encodings
+      def hadle_response(response)
+        if response["Content-Encoding"] == "gzip"
+          Zlib::GzipReader.new(StringIO.new(response.body)).read
+        elsif response["Content-Encoding"] == "deflate"
+          Zlib::Inflate.inflate(response.body)
+        else
+          response.body
+        end
+      end
+
+      # Fetch JSON data from a URL with custom headers
+      #
+      # @param [String] url The URL to fetch JSON data from
+      # @return [Hash] The parsed JSON data
+      def fetch_json_data(url)
+        uri = URI(url)
+
+        request = create_request(uri)
+
+        # Send the request and get the response
+        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request(request)
+        end
+
+        # Check if the response is successful
+        unless response.code.to_i == 200
+          Util.warn "Failed to fetch data: #{response.code} #{response.message}"
+          return {}
+        end
+
+        body = hadle_response(response)
+
+        # Parse the JSON response
+        JSON.parse(body)
+      rescue StandardError => e
+        # Handle any errors during the fetching process
+        Util.error "Error fetching JSON data from #{url}: #{e.message}"
+        {}
+      end
+
+      #
+      # Extract data for handbooks
+      #
+      def extract_handbooks_data
+        data = fetch_json_data(HANDBOOKS_URL)
+        Util.info "Extracting handbooks data..."
+        data["pageProps"]["handbooks"]["nodes"].each do |entry|
+          handbook = entry["handbook"]
+          versions = handbook["versions"]
+
+          description_parts = handbook["description"]&.split("<br />") || ["", ""]
+          title_en = description_parts[0].strip if description_parts[0]
+          abstract = description_parts[1].strip if description_parts[1]
+
+          doctype = entry["slug"].match("-") ?  "annex" : "handbook"
+
+          versions.each do |version|
+            item = HandbookParser.new(
+              version: version, entry: entry, title_en: title_en, abstract: abstract, doctype: doctype
+            ).parse
+            save_document(item)
+
+            # ::Relaton::Plateau::BibItem.new(
+            #   pubid: "PLATEAU Handbook ##{entry["slug"]}",
+            #   title_jp: handbook["title"],
+            #   title_en: title_en,
+            #   abstract_jp: abstract_jp,
+            #   cover: "https://www.mlit.go.jp/#{handbook["thumbnail"]["mediaItemUrl"]}",
+            #   type: document_type,
+            #   publication_date: Date.parse(version["date"].gsub(".", "-")),
+            #   url_pdf: version["pdf"],
+            #   url_html: version["html"],
+            #   filesize: version["filesize"].to_i,
+            #   edition_number: version["title"].match(/\d\.\d/)[0],
+            #   edition_text: version["title"],
+            #   # tags: [],
+            # )
+          end
+        end
+        index.save
+      end
+
+      #
+      # Extract data for technical reports
+      #
+      def extract_technical_reports_data
+        data = fetch_json_data(TECHNICAL_REPORTS_URL)
+        Util.info "Extracting technical reports data..."
+        data["pageProps"]["nodes"].map do |entry|
+          save_document(TechnicalReportParser.new(entry).parse)
+
+          # technical_report = entry["technicalReport"]
+
+          # ::Relaton::Plateau::BibItem.new(
+          #   title_jp: technical_report["title"],
+          #   abstract_jp: technical_report["subtitle"],
+          #   cover: "https://www.mlit.go.jp/#{technical_report["thumbnail"]["mediaItemUrl"]}",
+          #   pubid: "PLATEAU Tech Report ##{entry["slug"]}",
+          #   type: "technical-report",
+          #   subtype: entry["technicalReportCategories"]["nodes"].map { |cat| cat["name"] },
+          #   publication_date: Date.parse(entry["date"]),
+          #   url_pdf: technical_report["pdf"],
+          #   filesize: technical_report["filesize"].to_i,
+          #   edition_number: "1.0",
+          #   edition_text: "1.0",
+          #   tags: entry["globalTags"]["nodes"].map { |tag| tag["name"] },
+          # )
+
+        end
+        index.save
+      end
+
+      # def self.save_to_yaml(data, filename)
+      #   File.open(filename, "w") do |file|
+      #     file.write(data.to_yaml)
+      #   end
+      #   puts "Data saved to #{filename}."
+      # end
+
+      def save_document(item)
+        id = item.docidentifier.first.id
+        file = file_name id
+        if @files.include?(file)
+          Util.warn "File #{file} already exists, skipping.", key: id
+        else
+          File.write(file, serialize(item))
+          @files << file
+          index.add_or_update id, file
+        end
+      end
+
+      def file_name(id)
+        name = id.gsub(/\s+/, "_").gsub(/\W+/, "").downcase
+        if id.match?(/民間活用編/)
+          name += "_private"
+        elsif id.match?(/公共活用編/)
+          name += "_public"
+        end
+        File.join(@output, "#{name}.#{@ext}")
+      end
+
+      def serialize(item)
+        case @format
+        when "yaml" then item.to_hash.to_yaml
+        when "xml" then item.to_xml bibdata: true
+        else item.send("to_#{@format}")
+        end
+      end
+    end
+  end
+end

data/lib/relaton/plateau/handbook_parser.rb

@@ -0,0 +1,70 @@
+# encoding: UTF-8
+
+module Relaton
+  module Plateau
+    class HandbookParser < Parser
+      def initialize(version:, entry:, title_en:, abstract:, doctype:)
+        @version = version
+        @entry = entry
+        super entry["handbook"]
+        @title_en = title_en
+        @abstract = abstract
+        @doctype = doctype
+      end
+
+      private
+
+      def edition
+        @edition ||= @version["title"].split.first
+      end
+
+      def parse_docid
+        super << create_docid("PLATEAU Handbook ##{@entry["slug"]} #{edition}")
+      end
+
+      def parse_title
+        title = super
+        title << create_title(@title_en, "en", "Latn") if @title_en
+        title
+      end
+
+      def parse_abstract
+        abstr = super
+        abstr << create_formatted_string(@abstract) if @abstract
+        abstr
+      end
+
+      def parse_edition
+        number = edition.match(/\d\.\d/)[0]
+        RelatonBib::Edition.new(content: edition, number: number)
+      end
+
+      def parse_doctype
+        DocumentType.new type: @doctype
+      end
+
+      def parse_date
+        super << create_date(@version["date"].gsub(".", "-"))
+      end
+
+      def parse_link
+        %w[pdf html].map do |type|
+          next unless @version[type]
+
+          create_link(@version[type], type)
+        end.compact
+      end
+
+      def parse_filesize
+        @version["filesize"].to_i
+      end
+
+      def parse_structuredidentifier
+        strid = RelatonBib::StructuredIdentifier.new(
+          type: "Handbook", agency: ["PLATEAU"], docnumber: @entry["slug"], edition: edition
+        )
+        RelatonBib::StructuredIdentifierCollection.new [strid]
+      end
+    end
+  end
+end

data/lib/relaton/plateau/hash_converter.rb

@@ -0,0 +1,50 @@
+module Relaton
+  module Plateau
+    module HashConverter
+      include RelatonBib::HashConverter
+      extend self
+      # @param args [Hash]
+      # @return [Hash]
+      def hash_to_bib(args)
+        ret = super
+        return unless ret
+        return ret unless ret[:ext]
+
+        hash_to_bib_cover ret
+        hash_to_bib_filesize ret
+        hash_to_bib_stagename ret
+        ret.delete :ext
+        ret
+      end
+
+      def hash_to_bib_cover(ret)
+        return unless ret[:ext][:cover]
+
+        image = ret[:ext][:cover][:image]
+        ret[:cover] = Cover.new(RelatonBib::Image.new(**image))
+      end
+
+      def hash_to_bib_filesize(ret)
+        return unless ret[:ext][:filesize]
+
+        ret[:filesize] = ret[:ext][:filesize].to_i
+      end
+
+      def hash_to_bib_stagename(ret)
+        return unless ret[:ext][:stagename]
+
+        ret[:stagename] = Stagename.new(**ret[:ext][:stagename])
+      end
+
+      # @param item_hash [Hash]
+      # @return [RelatonCie::BibliographicItem]
+      def bib_item(item_hash)
+        BibItem.new(**item_hash)
+      end
+
+      def create_doctype(**args)
+        DocumentType.new(**args)
+      end
+    end
+  end
+end