relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/oiml.rb

@@ -0,0 +1,24 @@
+require "relaton/bib"
+require "relaton/index"
+require "pubid"
+require_relative "version"
+require_relative "oiml/util"
+require_relative "oiml/docidentifier"
+require_relative "oiml/item"
+require_relative "oiml/bibitem"
+require_relative "oiml/bibdata"
+require_relative "oiml/bibliography"
+
+module Relaton
+  module Oiml
+    INDEXFILE = "index-v2".freeze
+
+    class Error < StandardError; end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end

data/lib/relaton/omg/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Omg
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/omg/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Omg
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/omg/bibliography.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Omg
+    # OMG bibliography module
+    module Bibliography
+      extend self
+
+      # @param text [String] the OMG standard reference
+      # @return [Relaton::Omg::Item]
+      def search(text)
+        Scraper.scrape_page text
+      end
+
+      # @param code [String] the OMG standard reference
+      # @param year [String] the year the standard was published (optional)
+      # @param opts [Hash] options
+      # @return [Relaton::Omg::Item]
+      def get(code, _year = nil, _opts = {})
+        Util.info "Fetching from www.omg.org ...", key: code
+        result = search code
+        if result
+          Util.info "Found: `#{result.docidentifier.first.content}`", key: code
+        else
+          Util.info "Not found.", key: code
+        end
+        result
+      end
+    end
+  end
+end

data/lib/relaton/omg/docidentifier.rb

@@ -0,0 +1,67 @@
+module Relaton
+  module Omg
+    # An OMG document identifier backed by `Pubid::Omg`.
+    #
+    # `content` stays a plain string, so serialization is unchanged; the parsed
+    # identifier lives beside it in `@pubid` and drives the mutators. Follows
+    # the IALA and CEN shape (`lib/relaton/iala/docidentifier.rb`,
+    # `lib/relaton/cen/model/docidentifier.rb`).
+    class Docidentifier < Bib::Docidentifier
+      # @return [Pubid::Omg::Identifier, nil] nil when the content is not an
+      #   OMG identifier, or the grammar cannot read it
+      attr_reader :pubid
+
+      # Capture the inherited (LocalizedMarkedUpString) content setter before
+      # overriding #content=, so #refresh_content! writes the re-rendered string
+      # back WITHOUT re-parsing it and discarding the mutation.
+      alias_method :store_content, :content=
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          # `pubid` is required lazily because deserialization reaches this
+          # class without the flavor entry file having been loaded. LoadError
+          # degrades to a plain string; StandardError covers a non-OMG value
+          # and a title the grammar rejects, both of which are DATA and must
+          # not raise. A malformed *query* raises — see Scraper.scrape_page.
+          require "pubid"
+          ::Pubid::Omg::Identifier.parse(value)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+
+      # OMG identifiers carry no date. The version is OMG's discriminator
+      # (`OMG AMI4CCM 1.0` and `OMG AMI4CCM 1.1` are two editions of one
+      # specification), so the version-agnostic ("most recent") reference drops
+      # the version — as IALA maps `remove_date!` onto its edition.
+      def remove_date!
+        return unless @pubid
+
+        replace_pubid @pubid.exclude(:version)
+      end
+
+      # The part is the volume or format segment after the version, e.g.
+      # `Superstructure` in `OMG UML 2.1.1 Superstructure`.
+      def remove_part!
+        return unless @pubid
+
+        replace_pubid @pubid.exclude(:part)
+      end
+
+      # `to_all_parts!` stays the inherited no-op. An OMG part is a volume or a
+      # format name, not a numbered part, so there is no "all parts" form.
+
+      private
+
+      # `Pubid#exclude` returns a COPY, so the new identifier replaces the old
+      # one.
+      def replace_pubid(new_pubid)
+        @pubid = new_pubid
+        store_content @pubid.to_s
+      end
+    end
+  end
+end

data/lib/relaton/omg/ext.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Omg
+    class Ext < Bib::Ext
+      def get_schema_version = Relaton.schema_versions["relaton-model-omg"]
+    end
+  end
+end

data/lib/relaton/omg/item.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Omg
+    class Item < Bib::Item
+      model ItemData
+      attribute :docidentifier, Docidentifier, collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/omg/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Omg
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/omg/processor.rb

@@ -0,0 +1,35 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Omg
+    class Processor < Core::Processor
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_omg
+        @prefix = "OMG"
+        @pubid_flavor = :Omg # Pubid::Omg.prefixes is ["OMG"], the same as @prefix
+        @defaultprefix = /^OMG /
+        @idtype = "OMG"
+      end
+
+      def get(code, date, opts)
+        require_relative "../omg"
+        Bibliography.get(code, date, opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../omg"
+        Bibitem.from_xml xml
+      end
+
+      def from_yaml(yaml)
+        require_relative "../omg"
+        Item.from_yaml yaml
+      end
+
+      def grammar_hash
+        require_relative "../omg"
+        @grammar_hash ||= Omg.grammar_hash
+      end
+    end
+  end
+end

data/lib/relaton/omg/scraper.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require "json"
+require "mechanize"
+require "pubid"
+
+module Relaton
+  module Omg
+    class Scraper
+      URL_PATTERN = "https://www.omg.org/spec/"
+      LD_DATE = "https://www.omg.org/techprocess/ab/SpecificationMetadata/publicationDate"
+
+      # @param acronym [String] the specification acronym, e.g. "UML"
+      # @param version [String, nil] the version, e.g. "2.1.1" or "2.5 beta 1"
+      # @param part [String, nil] the document part, e.g. "Superstructure"
+      def initialize(acronym, version = nil, part = nil)
+        @acronym = acronym
+        @version = version
+        @part = part
+      end
+
+      # @param ref [String] the OMG reference, e.g. "OMG UML 2.1.1 Superstructure"
+      # @return [Relaton::Omg::ItemData, nil] nil when the page is not found
+      # @raise [Pubid::Errors::ParseError] when the reference is not an OMG
+      #   identifier
+      def self.scrape_page(ref)
+        pubid = ::Pubid::Omg::Identifier.parse(ref)
+        scraper = new(pubid.acronym, pubid.version, pubid.part)
+        doc = scraper.get_doc
+        return if doc.nil? || scraper.fetch_link.empty?
+
+        Omg::ItemData.new(**scraper.item)
+      end
+
+      def get_doc
+        @url = "#{URL_PATTERN}#{@acronym}/"
+        @url += @version.gsub(" ", "/") if @version
+        agent = Mechanize.new
+        agent.open_timeout = 10
+        @doc = agent.get(@url)
+      rescue Mechanize::ResponseCodeError => e
+        return if e.response_code == "404"
+
+        raise Relaton::RequestError, "Unable acces #{@url} (#{e.response_code})"
+      rescue Net::OpenTimeout
+        raise Relaton::RequestError, "Unable acces #{@url} (timeout)"
+      end
+
+      def item
+        {
+          fetched: ::Date.today.to_s,
+          docidentifier: fetch_docid,
+          title: fetch_title,
+          abstract: fetch_abstract,
+          version: fetch_version,
+          date: fetch_date,
+          status: fetch_status,
+          source: fetch_link,
+          relation: fetch_relation,
+          keyword: fetch_keyword,
+          license: fetch_license,
+        }
+      end
+
+      def fetch_title
+        content = @doc.at('//dt[.="Title:"]/following-sibling::dd').text
+        content += ": #{@part}" if @part
+        [Bib::Title.new(type: "main", content: content, language: "en", script: "Latn")]
+      end
+
+      # The version comes from the page, not from the query, so a versionless
+      # query (`OMG AMI4CCM`) answers with the version that the page shows.
+      def fetch_docid
+        id = render_id(@acronym, doc_version, @part)
+        [Docidentifier.new(content: id, type: "OMG", primary: true)]
+      end
+
+      def fetch_abstract
+        content = @doc.at('//section[@id="document-metadata"]/div/div/p').text
+        [Bib::Abstract.new(content: content, language: "en", script: "Latn")]
+      end
+
+      def fetch_version
+        [Bib::Version.new(revision_date: pub_date, draft: doc_version)]
+      end
+
+      def doc_version
+        @doc_version ||= @doc.at('//dt[.="Version:"]/following-sibling::dd/p/span').text
+      end
+
+      def fetch_date
+        return [] unless pub_date
+
+        [Bib::Date.new(type: "published", at: pub_date.to_s)]
+      end
+
+      def pub_date
+        return @pub_date if defined? @pub_date
+
+        @pub_date = jsonld_date || dd_date
+      end
+
+      # The visible date renders the month name in the locale of the server, and
+      # the CDN caches that variant. Parse the machine-readable value first.
+      #
+      # @return [Date, nil]
+      def jsonld_date
+        script = @doc.at('//script[@type="application/ld+json"]')
+        return unless script
+
+        node = JSON.parse(script.text).find { |e| e.is_a?(Hash) && e[LD_DATE] }
+        value = node && node[LD_DATE].first["@value"]
+        value && ::Date.parse(value)
+      rescue JSON::ParserError, ::Date::Error
+        nil
+      end
+
+      # @return [Date, nil]
+      def dd_date
+        text = @doc.at('//dt[.="Publication Date:"]/following-sibling::dd').text.strip
+        ::Date.parse text
+      rescue ::Date::Error
+        Util.warn "Cannot parse the publication date `#{text}`."
+        nil
+      end
+
+      def fetch_status
+        status = @doc.at('//dt[.="Document Status:"]/following-sibling::dd')
+        stage = status.text.strip.match(/\w+/).to_s
+        Bib::Status.new(stage: Bib::Status::Stage.new(content: stage))
+      end
+
+      def fetch_link
+        return @links if @links
+
+        @links = []
+        if @part
+          a = @doc.at("//a[@href='#{@url}/#{@part}/PDF']")
+          @links << Bib::Uri.new(type: "src", content: a[:href]) if a
+        else
+          a = @doc.at('//dt[.="This Document:"]/following-sibling::dd/a')
+          @links << Bib::Uri.new(type: "src", content: a[:href]) if a
+          pdf = @doc.at('//a[@class="download-document"]')
+          @links << Bib::Uri.new(type: "pdf", content: pdf[:href]) if pdf
+        end
+        @links
+      end
+
+      def fetch_relation
+        v = @doc.xpath('//h2[.="History"]/following-sibling::section/div/table/tbody/tr')
+        v.reduce([]) do |mem, row|
+          ver = row.at("td").text
+          unless ver == doc_version
+            acronym = row.at("td[3]/a")[:href].split("/")[4]
+            id = render_id(acronym, ver)
+            docid = Docidentifier.new(content: id, type: "OMG")
+            bibitem = Bib::ItemBase.new(formattedref: Bib::Formattedref.new(content: id), docidentifier: [docid])
+            mem << Bib::Relation.new(type: "obsoletes", bibitem: bibitem)
+          end
+          mem
+        end
+      end
+
+      def fetch_keyword
+        @doc.xpath('//dt[.="Categories:"]/following-sibling::dd/ul/li/a/em').map do |kw|
+          Bib::Keyword.new(vocab: Bib::LocalizedString.new(content: kw.text))
+        end
+      end
+
+      def fetch_license
+        @doc.xpath(
+          '//dt/span/a[contains(., "IPR Mode")]/../../following-sibling::dd/span',
+        ).map { |l| l.text.match(/[\w\s-]+/).to_s.strip }
+      end
+
+      private
+
+      # @return [String] the identifier, rendered by pubid
+      def render_id(acronym, version, part = nil)
+        ::Pubid::Omg::Identifiers::Specification.new(
+          acronym: acronym, version: version, part: part,
+        ).to_s
+      end
+    end
+  end
+end

data/lib/relaton/omg/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Omg
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-omg".freeze
+    end
+  end
+end

data/lib/relaton/omg.rb

@@ -0,0 +1,25 @@
+require "nokogiri"
+require "relaton/bib"
+require_relative "version"
+require_relative "omg/util"
+require_relative "omg/ext"
+require_relative "omg/docidentifier"
+require_relative "omg/item_data"
+require_relative "omg/item"
+require_relative "omg/bibitem"
+require_relative "omg/bibdata"
+require_relative "omg/scraper"
+require_relative "omg/bibliography"
+
+module Relaton
+  module Omg
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/plateau/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Plateau
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/plateau/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Plateau
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/plateau/bibliography.rb

@@ -0,0 +1,31 @@
+module Relaton
+  module Plateau
+    module Bibliography
+      extend self
+
+      def search(code)
+        HitCollection.new(code).find
+      end
+
+      # Only a transport failure is rescued, and it becomes the
+      # Relaton::RequestError that Relaton::Db retries. Anything else keeps
+      # its own class: an unrecognized reference raises Pubid::Errors::ParseError
+      # (relaton-cli reports it), and a bug keeps its backtrace.
+      def get(code, _year = nil, _opts = {}) # rubocop:disable Metrics/MethodLength
+        Util.info "Fetching ...", key: code
+        result = search(code).fetch_doc
+        if result
+          Util.info "Found `#{result.docidentifier.first.content}`", key: code
+          result
+        else
+          Util.warn "Not found.", key: code
+        end
+      rescue SocketError, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, Net::ReadTimeout, OpenSSL::SSL::SSLError,
+             Errno::ETIMEDOUT => e
+        raise Relaton::RequestError, e.message
+      end
+    end
+  end
+end

data/lib/relaton/plateau/data_fetcher.rb

@@ -0,0 +1,176 @@
+require "json"
+require "relaton/core"
+require_relative "../plateau"
+require_relative "parser"
+require_relative "handbook_parser"
+require_relative "technical_report_parser"
+
+module Relaton
+  module Plateau
+    # Fetcher class to fetch data from the Plateau website
+    class DataFetcher < Core::DataFetcher
+      HANDBOOKS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/handbooks.json".freeze
+      TECHNICAL_REPORTS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/technical-reports.json".freeze
+
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :plateau, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Plateau::Identifier
+        )
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def fetch(source)
+        case source
+        when "plateau-handbooks" then extract_handbooks_data
+        when "plateau-technical-reports" then extract_technical_reports_data
+        else puts "Invalid source: #{source}"
+        end
+      end
+
+      # Create a GET request with custom headers to mimic a browser
+      def create_request(uri)
+        request = Net::HTTP::Get.new(uri)
+        request["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0"
+        request["Accept"] = "*/*"
+        request["Accept-Language"] = "en-US,en;q=0.5"
+        request["Accept-Encoding"] = "gzip, deflate, br, zstd"
+        request["Referer"] = "https://www.mlit.go.jp/plateau/libraries/"
+        request["purpose"] = "prefetch"
+        request["x-nextjs-data"] = "1"
+        request["Connection"] = "keep-alive"
+        request
+      end
+
+      # Handle different content encodings
+      def hadle_response(response)
+        if response["Content-Encoding"] == "gzip"
+          Zlib::GzipReader.new(StringIO.new(response.body)).read
+        elsif response["Content-Encoding"] == "deflate"
+          Zlib::Inflate.inflate(response.body)
+        else
+          response.body
+        end
+      end
+
+      # Fetch JSON data from a URL with custom headers
+      #
+      # @param [String] url The URL to fetch JSON data from
+      # @return [Hash] The parsed JSON data
+      def fetch_json_data(url)
+        uri = URI(url)
+
+        request = create_request(uri)
+
+        # Send the request and get the response
+        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request(request)
+        end
+
+        # Check if the response is successful
+        unless response.code.to_i == 200
+          Util.warn "Failed to fetch data: #{response.code} #{response.message}"
+          return {}
+        end
+
+        body = hadle_response(response)
+
+        # Parse the JSON response
+        JSON.parse(body)
+      rescue StandardError => e
+        # Handle any errors during the fetching process
+        Util.error "Error fetching JSON data from #{url}: #{e.message}"
+        {}
+      end
+
+      #
+      # Extract data for handbooks
+      #
+      def extract_handbooks_data
+        data = fetch_json_data(HANDBOOKS_URL)
+        Util.info "Extracting handbooks data..."
+        data["pageProps"]["handbooks"]["nodes"].each do |entry|
+          handbook = entry["handbook"]
+          doctype = entry["slug"].match("-") ? "annex" : "handbook"
+
+          handbook["versions"].each do |version|
+            item = HandbookParser.new(version: version, entry: entry, doctype: doctype, errors: @errors).parse
+            save_document(item)
+          end
+        end
+        index.save
+        report_errors
+      end
+
+      #
+      # Extract data for technical reports
+      #
+      def extract_technical_reports_data
+        data = fetch_json_data(TECHNICAL_REPORTS_URL)
+        Util.info "Extracting technical reports data..."
+        data["pageProps"]["nodes"].map do |entry|
+          save_document(TechnicalReportParser.new(entry, @errors).parse)
+        end
+        index.save
+        report_errors
+      end
+
+      def save_document(item)
+        id = item.docidentifier.first.content
+        file = file_name id
+        if @files.include?(file)
+          Util.warn "File #{file} already exists, skipping.", key: id
+        else
+          File.write(file, serialize(item))
+          @files << file
+          pid = pubid id
+          if pid
+            index.add_or_update pid, file
+          else
+            Util.warn "Unparseable id `#{id}` was not indexed (#{file})", key: id
+          end
+        end
+      end
+
+      # Parse a canonical PLATEAU docidentifier into a Pubid::Plateau::Identifier,
+      # or nil if pubid can't parse it or it doesn't round-trip through
+      # `from_hash(to_hash)` — so a single bad id never aborts the crawl or
+      # corrupts index-v2 (the read side rejects an index whose rows don't
+      # deserialize). The parser emits canonical ids, so this should not skip
+      # anything; the guard is defensive.
+      def pubid(id)
+        pid = ::Pubid::Plateau.parse id
+        hash = pid.to_hash
+        return nil unless ::Pubid::Plateau::Identifier.from_hash(hash).to_hash == hash
+
+        pid
+      rescue StandardError
+        nil
+      end
+
+      def file_name(id)
+        name = id.gsub(/\s+/, "-").gsub(/[^\w-]+/, "").downcase
+        if id.match?(/民間活用編/)
+          name += "-private"
+        elsif id.match?(/公共活用編/)
+          name += "-public"
+        end
+        File.join(@output, "#{name}.#{@ext}")
+      end
+
+      def to_yaml(bib)
+        Item.to_yaml(bib)
+      end
+
+      def to_xml(bib)
+        Item.to_xml(bib, bibdata: true)
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+    end
+  end
+end

data/lib/relaton/plateau/doctype.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Plateau
+    class Doctype < Bib::Doctype
+      attribute :content, :string, values: %w[handbook technical-report annex]
+    end
+  end
+end

data/lib/relaton/plateau/ext.rb

@@ -0,0 +1,23 @@
+require_relative "doctype"
+
+module Relaton
+  module Plateau
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :stagename, Iso::Stagename
+      attribute :filesize, :integer
+
+      xml do
+        map_element "stagename", to: :stagename
+        map_element "filesize", to: :filesize
+      end
+
+      key_value do
+        map_element "stagename", to: :stagename
+        map_element "filesize", to: :filesize
+      end
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-plateau"]
+    end
+  end
+end