relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/plateau/handbook_parser.rb

@@ -0,0 +1,102 @@
+# encoding: UTF-8
+
+module Relaton
+  module Plateau
+    class HandbookParser < Parser
+      def initialize(version:, entry:, doctype:, errors: {})
+        @version = version
+        @entry = entry
+        super(entry["handbook"], errors)
+        @doctype = doctype
+      end
+
+      private
+
+      def edition
+        @edition ||= @version["title"].split.first.match(/[\d.]+/).to_s
+      end
+
+      # Canonical PLATEAU handbook edition label, e.g. "第3.0版" — the form
+      # Pubid::Plateau parses. The Latinized #edition is kept for the structured
+      # Bib::Edition/number and the ext structuredidentifier.
+      def edition_label
+        @edition_label ||= @version["title"].to_s[/第[\d.]+版/].to_s
+      end
+
+      def slug_number
+        @slug_number ||= @entry["slug"]&.to_s&.split("_")&.first
+      end
+
+      def parse_docnumber
+        @errors[:hb_docnumber] &&= @entry["slug"].nil? || @entry["slug"].to_s.empty?
+        ["Handbook ##{slug_number}", edition_label].reject { |s| s.to_s.empty? }.join(" ")
+      end
+
+      def parse_abstract
+        unless @item["description"]
+          @errors[:hb_abstract] &&= true
+          return []
+        end
+
+        result = @item["description"].split("<br />").filter_map do |part|
+          text = part.strip
+          next if text.empty?
+
+          lang, script = detect_lang(text)
+          create_abstract(text, lang, script)
+        end
+        @errors[:hb_abstract] &&= result.empty?
+        result
+      end
+
+      def parse_edition
+        if edition.nil? || edition.empty?
+          @errors[:hb_edition] &&= true
+          return
+        end
+
+        number = edition.match(/\d\.\d/)[0]
+        result = Bib::Edition.new(content: edition, number: number)
+        @errors[:hb_edition] &&= result.nil?
+        result
+      end
+
+      def parse_date
+        if @version["date"].nil? || @version["date"].empty?
+          @errors[:hb_date] &&= true
+          return super
+        end
+
+        result = super << create_date(@version["date"].gsub(".", "-"))
+        @errors[:hb_date] &&= result.empty?
+        result
+      end
+
+      def parse_source
+        result = %w[pdf html].map do |type|
+          next unless @version[type]
+
+          create_link(@version[type], type)
+        end.compact
+        @errors[:hb_source] &&= result.empty?
+        result
+      end
+
+      def parse_ext
+        strid = Bib::StructuredIdentifier.new(
+          type: "Handbook", agency: ["PLATEAU"], docnumber: slug_number, edition: edition
+        )
+        Ext.new(
+          doctype: Doctype.new(content: @doctype),
+          flavor: "plateau",
+          structuredidentifier: [strid],
+          filesize: filesize
+        )
+      end
+      def filesize
+        @errors[:hb_filesize] &&= @version["filesize"].nil?
+        @version["filesize"].to_i
+      end
+    end
+  end
+end

data/lib/relaton/plateau/hit.rb

@@ -0,0 +1,21 @@
+module Relaton
+  module Plateau
+    class Hit < Relaton::Core::Hit
+      # The index names the file, so a non-200 is an access failure, not a
+      # miss. Without the check the error page body reached Item.from_yaml,
+      # which returned an item with no docidentifier.
+      def item
+        @item ||= begin
+          uri = URI("#{HitCollection::ENDPOINT}#{hit[:file]}")
+          resp = Net::HTTP.get_response(uri)
+          unless resp.code == "200"
+            raise Relaton::RequestError,
+                  "Could not access #{uri}: HTTP #{resp.code}"
+          end
+
+          Item.from_yaml(resp.body)
+        end
+      end
+    end
+  end
+end

data/lib/relaton/plateau/hit_collection.rb

@@ -0,0 +1,79 @@
+require_relative "hit"
+
+module Relaton
+  module Plateau
+    class HitCollection < Relaton::Core::HitCollection
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-plateau/v2/"
+
+      # An edition suffix in either the canonical Japanese form (`第1.0版`) or the
+      # legacy Latin form (`1.0`), stripped in #to_all_editions to derive the
+      # edition-less family id from a relaton-bib docidentifier *string* (the
+      # fetched document is not a pubid). Matching itself is done on pubid objects.
+      EDITION_SUFFIX = / (?:第[\d.]+版|\d+\.\d+)$/
+
+      # An edition-less reference selects every edition of the document with
+      # pubid's subset match `pubid_ref === row`: the omitted edition matches
+      # any value, and pubid declares `annex` strict for PLATEAU, so
+      # `PLATEAU Handbook #03` does not reach `#03-1`. A reference with an
+      # edition selects that one row with `==`.
+      def find
+        @array = index.search(pubid_ref, exact: !all_editions?)
+          .map { |row| Hit.new(row, self) }
+        self
+      end
+
+      def fetch_doc
+        return unless any?
+
+        all_editions? ? to_all_editions : first.item
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :plateau, url: "#{ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+          pubid_class: ::Pubid::Plateau::Identifier
+        )
+      end
+
+      private
+
+      # `#ref` (the raw query string, from Core::HitCollection) parsed into a
+      # canonical Pubid::Plateau::Identifier (memoized). Canonical (`第1.0版`),
+      # edition-less (`#00`), and legacy Latin (`#00 1.0`) references all
+      # parse — pubid normalizes Latin input to the canonical id
+      # (metanorma/pubid #269) — so search accepts both forms. An unrecognized
+      # reference raises; like ISO and 3GPP we let it propagate — relaton-cli
+      # rescues Pubid::Errors::Error. See lib/relaton/plateau/CLAUDE.md.
+      def pubid_ref
+        @pubid_ref ||= ::Pubid::Plateau.parse(ref)
+      end
+
+      # A reference with no edition (`PLATEAU Handbook #00`, or any Technical
+      # Report — TRs carry no edition) asks for all editions of the document.
+      def all_editions?
+        pubid_ref.edition.nil?
+      end
+
+      def to_all_editions
+        return first.item if size < 2
+
+        bibitem = first.item
+        relations = map do |h|
+          Bib::Relation.new(type: "hasEdition", bibitem: h.item)
+        end
+        docid = bibitem.docidentifier.map do |d|
+          Bib::Docidentifier.new(
+            content: d.content.sub(EDITION_SUFFIX, ""), type: d.type, primary: d.primary
+          )
+        end
+        ItemData.new(
+          docidentifier: docid,
+          docnumber: bibitem.docnumber.sub(EDITION_SUFFIX, ""),
+          title: bibitem.title,
+          contributor: bibitem.contributor,
+          relation: relations
+        )
+      end
+    end
+  end
+end

data/lib/relaton/plateau/item.rb

@@ -0,0 +1,12 @@
+require_relative "ext"
+require_relative "item_data"
+
+module Relaton
+  module Plateau
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/plateau/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Plateau
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/plateau/parser.rb

@@ -0,0 +1,101 @@
+module Relaton
+  module Plateau
+    # Base class for Plateau parsers
+    class Parser
+      ATTRIS = %i[docidentifier docnumber title abstract depiction edition type
+                  date source contributor keyword ext].freeze
+
+      def initialize(item, errors = {})
+        @item = item
+        @errors = errors
+      end
+
+      def parse
+        args = ATTRIS.each_with_object({}) do |attr, hash|
+          hash[attr] = send("parse_#{attr}")
+        end
+        ItemData.new(**args)
+      end
+
+      private
+
+      def parse_docidentifier
+        result = [create_docid("PLATEAU #{parse_docnumber}")]
+        @errors[:parse_docidentifier] &&= result.empty?
+        result
+      end
+
+      def parse_docnumber; end
+
+      def create_docid(id)
+        Bib::Docidentifier.new(type: "PLATEAU", content: id, primary: true)
+      end
+
+      def create_abstract(content, lang = "ja", script = "Jpan")
+        Bib::Abstract.new(content: content, language: lang, script: script)
+      end
+
+      def detect_lang(text)
+        if text&.match?(/[\p{Han}\p{Katakana}\p{Hiragana}]/)
+          ["ja", "Jpan"]
+        else
+          ["en", "Latn"]
+        end
+      end
+
+      def parse_title
+        lang, script = detect_lang(@item["title"])
+        result = [create_title(@item["title"], lang, script)]
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      def create_title(title, lang, script)
+        Bib::Title.new(type: "main", content: title, language: lang, script: script)
+      end
+
+      def parse_abstract; [] end
+
+      def parse_depiction
+        image_ext = @item["thumbnail"]["mediaItemUrl"].split(".").last
+        mimetype = "image/"
+        mimetype += image_ext == "jpg" ? "jpeg" : image_ext
+        src = "https://www.mlit.go.jp/#{@item["thumbnail"]["mediaItemUrl"]}"
+        image = Bib::Image.new(src: src, mimetype: mimetype)
+        result = Bib::Depiction.new(scope: "cover", image: [image])
+        @errors[:parse_depiction] &&= result.nil?
+        [result]
+      end
+
+      def parse_edition; raise "Not implemented" end
+      def parse_type; "standard" end
+      def parse_date; [] end
+      def parse_source; [] end
+
+      def parse_contributor
+        name = [
+          Bib::TypedLocalizedString.new(content: "国土交通省", language: "ja", script: "Jpan"),
+          Bib::TypedLocalizedString.new(
+            content: "Japanese Ministry of Land, Infrastructure, Transport and Tourism",
+            language: "en", script: "Latn"
+          ),
+        ]
+        org = Bib::Organization.new(name: name, abbreviation: Bib::LocalizedString.new(content: "MLIT"))
+        result = [Bib::Contributor.new(organization: org, role: [Bib::Contributor::Role.new(type: "publisher")])]
+        @errors[:parse_contributor] &&= result.empty?
+        result
+      end
+
+      def create_date(date, type = "published")
+        Bib::Date.new(type: type, at: date)
+      end
+
+      def create_link(url, type)
+        Bib::Uri.new(type: type, content: url)
+      end
+
+      def parse_keyword; [] end
+      def parse_ext; raise "Not implemented" end
+    end
+  end
+end

data/lib/relaton/plateau/processor.rb

@@ -0,0 +1,51 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Plateau
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_plateau
+        @prefix = "PLATEAU"
+        @defaultprefix = /^PLATEAU\s/
+        @idtype = "PLATEAU"
+        @datasets = %w[plateau-handbooks plateau-technical-reports]
+      end
+
+      def get(code, date, opts)
+        require_relative "../plateau"
+        Bibliography.get(code, date, opts)
+      end
+
+      def fetch_data(source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(source, **opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../plateau"
+        Item.from_xml xml
+      end
+
+      def from_yaml(yaml)
+        require_relative "../plateau"
+        Item.from_yaml yaml
+      end
+
+      def grammar_hash
+        require_relative "../plateau"
+        @grammar_hash ||= ::Relaton::Plateau.grammar_hash
+      end
+
+      def threads = 3
+
+      def remove_index_file
+        require_relative "../plateau"
+        Relaton::Index.find_or_create(
+          :plateau, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/plateau/technical_report_parser.rb

@@ -0,0 +1,98 @@
+module Relaton
+  module Plateau
+    class TechnicalReportParser < Parser
+      def initialize(entry, errors = {})
+        @entry = entry
+        super(entry["technicalReport"], errors)
+      end
+
+      private
+
+      def parse_docnumber
+        @errors[:tr_docnumber] &&= @entry["slug"].nil? || @entry["slug"].to_s.empty?
+        "Technical Report ##{canonical_slug}"
+      end
+
+      # Canonical sub-number separator is "-" (Pubid::Plateau); source slugs use
+      # "_" (e.g. "46_1" -> "46-1"). PLATEAU Technical Report ids carry no
+      # edition in canonical form, so #edition_number is not part of the id.
+      def canonical_slug
+        @entry["slug"].to_s.tr("_", "-")
+      end
+
+      def parse_abstract
+        if @item["subtitle"].nil? || @item["subtitle"].empty?
+          @errors[:tr_abstract] &&= true
+          return super
+        end
+
+        result = super << create_abstract(@item["subtitle"])
+        @errors[:tr_abstract] &&= result.empty?
+        result
+      end
+
+      def parse_edition
+        result = Bib::Edition.new(content: edition_number, number: edition_number)
+        result
+      end
+
+      def edition_number = "1.0"
+
+      def parse_date
+        if @entry["date"].nil? || @entry["date"].empty?
+          @errors[:tr_date] &&= true
+          return super
+        end
+
+        date_str = @entry["date"].sub(/T.*/, "")
+        result = super << create_date(date_str)
+        @errors[:tr_date] &&= result.empty?
+        result
+      end
+
+      def parse_source
+        if @item["pdf"].nil? || @item["pdf"].empty?
+          @errors[:tr_source] &&= true
+          return super
+        end
+
+        result = super << create_link(@item["pdf"], "pdf")
+        @errors[:tr_source] &&= result.empty?
+        result
+      end
+
+      def parse_keyword
+        result = @entry["globalTags"]["nodes"].map do |tag|
+          Bib::Keyword.new(vocab: Bib::LocalizedString.new(content: tag["name"]))
+        end
+        @errors[:tr_keyword] &&= result.empty?
+        result
+      end
+
+      def parse_ext
+        strid = Bib::StructuredIdentifier.new(
+          type: "Technical Report", klass: parse_subdoctype,
+          agency: ["PLATEAU"], docnumber: @entry["slug"]
+        )
+        Ext.new(
+          doctype: Doctype.new(content: "technical-report"),
+          subdoctype: parse_subdoctype,
+          flavor: "plateau",
+          structuredidentifier: [strid],
+          filesize: filesize
+        )
+      end
+
+      def filesize
+        @errors[:tr_filesize] &&= @item["filesize"].nil?
+        @item["filesize"].to_i
+      end
+
+      def parse_subdoctype
+        result = @entry["technicalReportCategories"]["nodes"].dig(0, "name")
+        @errors[:tr_subdoctype] &&= result.nil?
+        result
+      end
+    end
+  end
+end

data/lib/relaton/plateau/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Plateau
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-plateau".freeze
+    end
+  end
+end

data/lib/relaton/plateau.rb

@@ -0,0 +1,29 @@
+require "net/http"
+require "uri"
+require "pubid"
+require "relaton/index"
+require "relaton/iso"
+require_relative "version"
+require_relative "plateau/util"
+# require_relative "plateau/docidentifier"
+# require_relative "plateau/document_type"
+require_relative "plateau/item"
+require_relative "plateau/bibitem"
+require_relative "plateau/bibdata"
+require_relative "plateau/hit_collection"
+require_relative "plateau/bibliography"
+require_relative "plateau/processor"
+# require_relative "plateau/xml_parser"
+# require_relative "plateau/hash_converter"
+
+module Relaton
+  module Plateau
+    INDEXFILE = "index-v2"
+
+    class Error < StandardError; end
+
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/sdo/config.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Sdo
+    # Configuration for the SDO org/logo store: where the index manifest lives,
+    # where to cache it, and for how long. Mirrors Relaton::Index::Config; the
+    # storage backend is swappable (e.g. an S3-backed object) via `storage=`.
+    class Config
+      # Default published index of the relaton-data-sdo data repo.
+      DEFAULT_URL =
+        "https://raw.githubusercontent.com/relaton/relaton-data-sdo/main/index.yaml"
+
+      attr_accessor :url, :storage, :storage_dir, :ttl
+
+      def initialize
+        @url = DEFAULT_URL
+        @storage = Relaton::Index::FileStorage
+        @storage_dir = File.join(Dir.home, ".relaton", "sdo")
+        @ttl = 24 * 60 * 60 # seconds
+      end
+    end
+
+    class << self
+      def configuration
+        @configuration ||= Config.new
+      end
+
+      def configure
+        yield configuration if block_given?
+        configuration
+      end
+    end
+  end
+end

data/lib/relaton/sdo/fetcher.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Sdo
+    # Download helper. The index manifest is text-cached under the configured
+    # storage dir with a TTL (reusing Relaton::Index::FileStorage); logo binaries
+    # are fetched on demand and left to in-process memoization on the Logo.
+    module Fetcher
+      module_function
+
+      # Fetch the index manifest, serving a fresh on-disk copy when present and
+      # falling back to a stale cache if the network fetch fails.
+      def fetch_index
+        config = Sdo.configuration
+        cache = File.join(config.storage_dir, "index.yaml")
+
+        if fresh?(cache, config)
+          cached = config.storage.read(cache)
+          return cached if cached
+        end
+
+        data = download(config.url)
+        return config.storage.read(cache) if data.nil? # stale cache or nil
+
+        config.storage.write(cache, data)
+        data
+      end
+
+      # GET a URL and return the body, or nil on failure. A URL with no scheme
+      # (or a file:// URL) is read straight off disk — handy for local fixtures.
+      def download(url)
+        uri = url.is_a?(URI) ? url : URI.parse(url.to_s)
+        return read_file(uri, url) if uri.scheme.nil? || uri.scheme == "file"
+
+        response = Net::HTTP.get_response(uri)
+        response.is_a?(Net::HTTPSuccess) ? response.body : nil
+      rescue StandardError
+        nil
+      end
+
+      def fresh?(file, config)
+        ctime = config.storage.ctime(file)
+        ctime && (Time.now - ctime) < config.ttl
+      end
+
+      def read_file(uri, url)
+        path = uri.path.to_s.empty? ? url.to_s : uri.path
+        File.exist?(path) ? File.binread(path) : nil
+      end
+    end
+  end
+end