relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.1

data/lib/relaton/w3c/data_parser.rb

@@ -0,0 +1,379 @@
+module Relaton
+  module W3c
+    class DataParser
+      include Relaton::W3c::SafeRealize
+
+      USED_TYPES = %w[WD NOTE PER PR REC CR].freeze
+
+      DOCTYPES = {
+        "TR" => "technicalReport",
+        "NOTE" => "groupNote",
+      }.freeze
+
+      STAGES = {
+        "RET" => "Retired",
+        "SPSD" => "Superseded Recommendation",
+        "OBSL" => "Obsoleted Recommendation",
+        "WD" => "Working Draft",
+        "CRD" => "Candidate Recommendation Draft",
+        "CR" => "Candidate Recommendation",
+        "PR" => "Proposed Recommendation",
+        "PER" => "Proposed Edited Recommendation",
+        "REC" => "Recommendation",
+      }.freeze
+
+      #
+      # Document parser initalization
+      #
+      # @param [W3cApi::Models::SpecVersion] spec
+      #
+      ERROR_KEYS = %i[status title doc_uri formattedref series date
+                      relation contributor doctype].freeze
+
+      def initialize(spec, errors = {})
+        @spec = spec
+        @errors = errors
+        ERROR_KEYS.each { |k| @errors[k] = true unless @errors.key?(k) }
+      end
+
+      #
+      # Initialize document parser and run it
+      #
+      # @param [W3cApi::Models::SpecVersion] spec
+      #
+      # @return [Relaton::W3c::ItemData, nil] bibliographic item
+      #
+      def self.parse(spec, errors = {})
+        new(spec, errors).parse
+      end
+
+      #
+      # Parse document
+      #
+      # @return [Relaton::W3c::ItemData] bibliographic item
+      #
+      def parse # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        ItemData.new(
+          type: "standard",
+          language: ["en"],
+          script: ["Latn"],
+          status: parse_status,
+          title: parse_title,
+          source: parse_source,
+          docidentifier: parse_docid,
+          formattedref: parse_formattedref,
+          docnumber: identifier,
+          series: parse_series,
+          date: parse_date,
+          relation: parse_relation,
+          contributor: parse_contrib,
+          ext: parse_ext,
+        )
+      end
+
+      #
+      # Parse ext with doctype
+      #
+      # @return [Ext, nil] ext
+      #
+      def parse_ext
+        dt = parse_doctype
+        result = Ext.new(doctype: dt, flavor: "w3c")
+        result
+      end
+
+      #
+      # Extract document status
+      #
+      # @return [Bib::Status, nil] document status
+      #
+      def parse_status
+        result = if @spec.respond_to?(:status) && @spec.status
+                   Bib::Status.new(stage: Bib::Status::Stage.new(content: @spec.status))
+                 end
+        @errors[:status] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse title
+      #
+      # @return [Array<Bib::Title>] title
+      #
+      def parse_title(spec = @spec)
+        return [] unless spec&.title && spec.title.strip != ""
+
+        result = [Bib::Title.new(content: spec.title, language: "en", script: "Latn")]
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      def doc_uri(spec = @spec)
+        result = spec.respond_to?(:uri) ? spec.uri : spec.shortlink
+        @errors[:doc_uri] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse link
+      #
+      # @return [Array<Bib::Uri>] link
+      #
+      def parse_source
+        [Bib::Uri.new(type: "src", content: doc_uri)]
+      end
+
+      #
+      # Parse docidentifier
+      #
+      # @return [Array<Bib::Docidentifier>] docidentifier
+      #
+      def parse_docid
+        id = pub_id(doc_uri)
+        [Bib::Docidentifier.new(type: "W3C", content: id, primary: true)]
+      end
+
+      #
+      # Generate PubID
+      #
+      # @return [String] PubID
+      #
+      def pub_id(url)
+        "W3C #{identifier(url)}"
+      end
+
+      #
+      # Generate identifier from URL
+      #
+      # @param [String] link
+      #
+      # @return [String] identifier
+      #
+      def identifier(link = doc_uri)
+        self.class.parse_identifier(link)
+      end
+
+      #
+      # Parse identifier from URL
+      #
+      # @param [String] url URL
+      #
+      # @return [String] identifier
+      #
+      def self.parse_identifier(url)
+        if /.+\/(\w+(?:[-+][\w.]+)+(?:\/\w+)?)/ =~ url.to_s
+          $1.to_s
+        else url.to_s.split("/").last
+        end
+      end
+
+      #
+      # Parse series
+      #
+      # @return [Array<Bib::Series>] series
+      #
+      def parse_series
+        result = if type
+                   title = Bib::Title.new(content: "W3C #{type}", language: "en", script: "Latn")
+                   [Bib::Series.new(title: [title], number: identifier)]
+                 else
+                   []
+                 end
+        @errors[:series] &&= result.empty?
+        result
+      end
+
+      #
+      # Extract type
+      #
+      # @return [String] type
+      #
+      def type
+        @type ||= @spec.respond_to?(:status) ? @spec.status : "technicalReport"
+      end
+
+      #
+      # Parse doctype
+      #
+      # @return [Doctype, nil] doctype
+      #
+      def parse_doctype
+        t = DOCTYPES[type] || DOCTYPES[type_from_link]
+        result = Doctype.new(content: t) if t
+        @errors[:doctype] &&= result.nil?
+        result
+      end
+
+      #
+      # Fetch type from link
+      #
+      # @return [String, nil] type
+      #
+      def type_from_link
+        @spec.shortlink.strip.match(/www\.w3\.org\/(TR)/)&.to_a&.fetch 1
+      end
+
+      #
+      # Parse date
+      #
+      # @return [Array<Bib::Date>] date
+      #
+      def parse_date
+        result = if @spec.respond_to?(:date)
+                   [Bib::Date.new(type: "published", at: @spec.date.to_date.to_s)]
+                 else
+                   []
+                 end
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse relation
+      #
+      # @return [Array<Bib::Relation>] relation
+      #
+      def parse_relation
+        result = if @spec.links.respond_to?(:version_history)
+                   version_history = realize @spec.links.version_history
+                   version_history&.links&.spec_versions&.map { |version| create_relation(version, "hasEdition") } || []
+                 else
+                   relations
+                 end
+        result = result.compact
+        @errors[:relation] &&= result.empty?
+        result
+      end
+
+      #
+      # Create relations
+      #
+      # @return [Array<Bib::Relation>] relations
+      #
+      def relations # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        rels = []
+        rels << create_relation(@spec.links.specification, "editionOf") if @spec.links.respond_to?(:specification)
+        if @spec.links.respond_to?(:predecessor_versions) && @spec.links.predecessor_versions
+          predecessor_versions = realize @spec.links.predecessor_versions
+          predecessor_versions&.links&.predecessor_versions&.each do |version|
+            rels << create_relation(version, "obsoletes")
+          end
+        end
+        if @spec.links.respond_to?(:successor_versions) && @spec.links.successor_versions
+          successor_versions = realize @spec.links.successor_versions
+          successor_versions&.links&.successor_versions&.each do |version|
+            rels << create_relation(version, "updatedBy", "errata")
+          end
+        end
+        rels.compact
+      end
+
+      #
+      # Create relation
+      #
+      # @param [Object] version version link
+      # @param [String] type relation type
+      # @param [String, nil] desc relation description
+      #
+      # @return [Bib::Relation] relation
+      #
+      def create_relation(version, type, desc = nil)
+        version_spec = realize version
+        return nil unless version_spec
+
+        url = doc_uri(version_spec)
+        id = pub_id(url)
+        title = parse_title(version_spec)
+        docid = Bib::Docidentifier.new(type: "W3C", content: id, primary: true)
+        link = [Bib::Uri.new(type: "src", content: url)]
+        bib = ItemData.new(title: title, docidentifier: [docid], source: link)
+        dsc = Bib::LocalizedMarkedUpString.new(content: desc) if desc
+        Bib::Relation.new(type: type, bibitem: bib, description: dsc)
+      end
+
+      #
+      # Parse formattedref
+      #
+      # @return [String, nil] formattedref
+      #
+      def parse_formattedref
+        result = if @spec.respond_to?(:uri)
+                   Bib::Formattedref.new(content: pub_id(@spec.uri))
+                 end
+        @errors[:formattedref] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse contributor
+      #
+      # @return [Array<Bib::Contributor>] contributor
+      #
+      def parse_contrib # rubocop:disable Metrics/MethodLength
+        contribs = [Bib::Contributor.new(
+          organization: create_w3c_org,
+          role: [Bib::Contributor::Role.new(type: "publisher")],
+        )]
+
+        if @spec.links.respond_to?(:editors)
+          editors = realize @spec.links.editors
+          editors&.links&.editors&.each do |ed|
+            editor = create_editor(ed)
+            contribs << editor if editor
+          end
+        end
+
+        result = contribs + parse_deliverers
+        @errors[:contributor] &&= result.empty?
+        result
+      end
+
+      def create_editor(unrealized_editor)
+        editor = realize unrealized_editor
+        return unless editor
+
+        surname = Bib::LocalizedString.new(content: editor.family, language: "en", script: "Latn")
+        forename = Bib::FullNameType::Forename.new(content: editor.given, language: "en", script: "Latn")
+        name = Bib::FullName.new(surname: surname, forename: [forename])
+        person = Bib::Person.new(name: name)
+        Bib::Contributor.new(
+          person: person,
+          role: [Bib::Contributor::Role.new(type: "editor")],
+        )
+      end
+
+      #
+      # Parse deliverers as contributors with role "author" and description "committee"
+      #
+      # @return [Array<Bib::Contributor>] deliverer contributors
+      #
+      def parse_deliverers # rubocop:disable Metrics/MethodLength
+        return [] unless @spec.links.respond_to?(:deliverers)
+
+        deliverers = realize @spec.links.deliverers
+        return [] unless deliverers&.links&.deliverers
+
+        deliverers.links.deliverers.map do |edg|
+          org = create_w3c_org.tap do |o|
+            subdiv_name = Bib::TypedLocalizedString.new(content: edg.title)
+            subdiv = Bib::Subdivision.new(name: [subdiv_name], type: "technical-committee")
+            o.subdivision = [subdiv]
+          end
+          role = Bib::Contributor::Role.new(
+            type: "author",
+            description: [Bib::LocalizedMarkedUpString.new(content: "committee")],
+          )
+          Bib::Contributor.new(organization: org, role: [role])
+        end
+      end
+
+      def create_w3c_org
+        Bib::Organization.new(
+          name: [Bib::TypedLocalizedString.new(content: "World Wide Web Consortium")],
+          abbreviation: Bib::LocalizedString.new(content: "W3C"),
+          uri: Bib::Uri.new(content: "https://www.w3.org"),
+        )
+      end
+    end
+  end
+end

data/lib/relaton/w3c/doctype.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module W3c
+    class Doctype < Bib::Doctype
+      attribute :content, :string, values: %w[groupNote technicalReport]
+    end
+  end
+end

data/lib/relaton/w3c/ext.rb

@@ -0,0 +1,11 @@
+require_relative "doctype"
+
+module Relaton
+  module W3c
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-w3c"]
+    end
+  end
+end

data/lib/relaton/w3c/item.rb

@@ -0,0 +1,12 @@
+require_relative "item_data"
+require_relative "ext"
+
+module Relaton
+  module W3c
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/w3c/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module W3c
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/w3c/processor.rb

@@ -0,0 +1,49 @@
+require "relaton/core/processor"
+
+module Relaton
+  module W3c
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_w3c
+        @prefix = "W3C"
+        @defaultprefix = %r{^W3C\s}
+        @idtype = "W3C"
+        @datasets = %w[w3c-api]
+      end
+
+      def get(code, date, opts)
+        require_relative "../w3c"
+        Bibliography.get(code, date, opts)
+      end
+
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../w3c"
+        Item.from_xml(xml)
+      end
+
+      def from_yaml(yaml)
+        require_relative "../w3c"
+        Item.from_yaml(yaml)
+      end
+
+      def grammar_hash
+        require_relative "../w3c"
+        @grammar_hash ||= Relaton::W3c.grammar_hash
+      end
+
+      def remove_index_file
+        require_relative "../w3c"
+        Relaton::Index.find_or_create(
+          :W3C, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/w3c/pubid.rb

@@ -0,0 +1,73 @@
+module Relaton
+  module W3c
+    class PubId
+      PARTS = %i[code stage type year date suff].freeze
+
+      PARTS.each { |part| attr_accessor part }
+
+      def initialize(**parts)
+        PARTS.each { |part| send "#{part}=", parts[part] }
+      end
+
+      #
+      # Parse document identifier.
+      #
+      # @param [String] docnumber document identifier
+      #
+      # @return [Relaton::W3c::PubId] document identifier parts
+      #
+      def self.parse(docnumber) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        %r{
+          (?:^|/)(?:(?:(?<stage>WD|CRD|CR|PR|PER|REC|SPSD|OBSL|RET)|(?<type>D?NOTE|TR))[\s/-])?
+          (?<code>\w+(?:[+-][\w.]+)*?)
+          (?:-(?<year>(?:18|19|20)\d{2}))?
+          (?:-(?<date>\d{8}|\d{6}|\d{4}))?
+          (?:/(?<suff>\w+))?(?:$|/)
+        }xi =~ docnumber
+        entry = { code: code }
+        entry[:stage] = stage if stage
+        entry[:type] = type if type && type != "TR"
+        entry[:year] = year if year
+        entry[:date] = date if date
+        entry[:suff] = suff if suff
+        new(**entry)
+      end
+
+      #
+      # Compare document identifiers against Hash ID representation.
+      #
+      # @param [Hash] other hash of document identifier parts
+      #
+      # @return [Boolean] true if document identifiers are same
+      #
+      def ==(other) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        other[:code].casecmp?(code) && other[:stage] == stage && other[:type] == type &&
+          (year.nil? || other[:year] == year) && (date.nil? || other[:date] == date) &&
+          (suff.nil? || other[:suff]&.casecmp?(suff))
+      end
+
+      #
+      # Convert docidentifier identifier to hash.
+      #
+      # @return [Hash] hash of docidentifier parts
+      #
+      def to_hash
+        PARTS.each_with_object({}) { |part, hash| hash[part] = send part if send part }
+      end
+
+      #
+      # Render docidentifier as a string, the inverse of {.parse}.
+      #
+      # @return [String] document identifier
+      #
+      def to_s
+        prefix = stage || (type if type && type != "TR")
+        base = [prefix, code].compact.join("-")
+        base = "#{base}-#{year}" if year
+        base = "#{base}-#{date}" if date
+        base = "#{base}/#{suff}" if suff
+        base
+      end
+    end
+  end
+end

data/lib/relaton/w3c/safe_realize.rb

@@ -0,0 +1,59 @@
+require "concurrent/map"
+
+module Relaton
+  module W3c
+    # Thin wrapper over lutaml-hal's `realize`. Successful objects are cached by
+    # w3c_api (it caches realized objects keyed by URL), so this only remembers
+    # resources that failed terminally and returns nil for them — so one broken
+    # link doesn't abort the crawl and isn't re-fetched on every reference.
+    #
+    # Transient failures are retried upstream: w3c_api retries HTTP 403 (the
+    # W3C rate-limit signal) and connection/timeout errors, and lutaml-hal
+    # retries 429 and 5xx. By the time an error surfaces here it is terminal.
+    module SafeRealize
+      # Hrefs that failed terminally — one map shared by every includer
+      # (DataFetcher and DataParser) since a broken resource is broken for the
+      # whole crawl. Initialized eagerly (at load, single-threaded) so the
+      # parallel fetcher's first concurrent access can't race a lazy `||=`;
+      # Concurrent::Map then handles the concurrent reads/writes.
+      @skipped = Concurrent::Map.new
+
+      def self.skipped
+        @skipped
+      end
+
+      # @param parent_resource [Object, nil] the index/page the link came from.
+      #   When the page was fetched with `embed: true`, its inlined `_embedded`
+      #   payload lets the link realize from memory instead of issuing an HTTP
+      #   request. nil (the default) preserves the plain remote-fetch behavior.
+      def realize(obj, parent_resource: nil)
+        href = resolve_href(obj)
+        return nil if SafeRealize.skipped.key?(href)
+
+        obj.realize(parent_resource: parent_resource)
+      rescue Lutaml::Hal::ConnectionError, Lutaml::Hal::TimeoutError, Faraday::Error, Net::OpenTimeout => e
+        # Network-level failure (already retried by w3c_api). The resource itself
+        # is fine, so don't skip it permanently — a later reference can try again.
+        Util.warn "Failed to realize object: #{href}, error: #{e.message}"
+        nil
+      rescue Lutaml::Hal::NotFoundError
+        Util.warn "Object not found: #{href}"
+        SafeRealize.skipped[href] = true
+        nil
+      rescue Lutaml::Hal::Error => e
+        # Definitive upstream error (403 rate-limit, 5xx, 429) already retried by
+        # w3c_api / lutaml-hal. Skip the broken/unavailable resource rather than
+        # re-hitting it for every link that references it.
+        Util.warn "Skipping #{href}, upstream error after retries: #{e.message}"
+        SafeRealize.skipped[href] = true
+        nil
+      end
+
+      private
+
+      def resolve_href(obj)
+        obj.href || obj.links.self.href
+      end
+    end
+  end
+end

data/lib/relaton/w3c/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module W3c
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-w3c".freeze
+    end
+  end
+end

data/lib/relaton/w3c.rb

@@ -0,0 +1,25 @@
+require "relaton/bib"
+require "relaton/index"
+require_relative "version"
+require_relative "w3c/util"
+require_relative "w3c/item"
+require_relative "w3c/bibitem"
+require_relative "w3c/bibdata"
+require_relative "w3c/bibliography"
+
+module Relaton
+  module W3c
+    INDEXFILE = "index-v1".freeze
+
+    class Error < StandardError; end
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/xsf/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Xsf
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/xsf/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Xsf
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/xsf/bibliography.rb

@@ -0,0 +1,24 @@
+module Relaton
+  module Xsf
+    module Bibliography
+      extend self
+
+      def search(ref)
+        HitCollection.new(ref).search
+      end
+
+      def get(code, _year = nil, _opts = {})
+        Util.info "Fetching from Relaton repository ...", key: code
+        result = search(code)
+        if result.empty?
+          Util.info "Not found.", key: code
+          return
+        end
+
+        bib = result.first.item
+        Util.info "Found: `#{bib.docidentifier.first.content}`", key: code
+        bib
+      end
+    end
+  end
+end