relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/ietf/bibxml_parser.rb

@@ -0,0 +1,232 @@
+module Relaton
+  module Ietf
+    module BibXMLParser # rubocop:disable Metrics/ModuleLength
+      extend self
+
+      def parse(xml)
+        reference = Rfcxml::V3::Reference.from_xml(xml)
+        FromRfcxml.new(reference).transform
+      end
+
+      def parse_rfc(xml)
+        rfc = Rfcxml::V3::Rfc.from_xml(xml)
+        FromRfc.new(rfc).transform
+      end
+
+      def pubid_type(id)
+        pref = id.match(/^(\S+)/)[1]
+        case pref
+        when "BCP", "FYI", "STD", "RFC" then "RFC"
+        when "I-D" then "Internet-Draft"
+        else "IETF"
+        end
+      end
+
+      def full_name_org(name) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
+        case name
+        when "ISO"
+          build_org(name, "International Organization for Standardization")
+        when "IAB"
+          build_org(name, "Internet Architecture Board")
+        when "IESG"
+          build_org(name, "Internet Engineering Steering Group")
+        when "IANA"
+          build_org(name, "Internet Assigned Numbers Authority")
+        when "International Organization for Standardization"
+          build_org("ISO", name)
+        when "Federal Networking Council", "Internet Architecture Board", "Internet Activities Board",
+          "Defense Advanced Research Projects Agency", "National Science Foundation",
+          "National Research Council", "National Bureau of Standards",
+          "Internet Engineering Steering Group"
+          abbr = name.split.map { |w| w[0] if w[0] == w[0].upcase }.join
+          build_org(abbr, name)
+        when "IETF Secretariat"
+          build_org("IETF", name)
+        when "Audio-Video Transport Working Group", /North American Directory Forum/, "EARN Staff",
+          "Vietnamese Standardization Working Group", "ACM SIGUCCS", "ESCC X.500/X.400 Task Force",
+          "Sun Microsystems", "NetBIOS Working Group in the Defense Advanced Research Projects Agency",
+          "End-to-End Services Task Force", "Network Technical Advisory Group", "Bolt Beranek",
+          "Newman Laboratories", "Gateway Algorithms and Data Structures Task Force",
+          "Network Information Center. Stanford Research Institute", "RFC Editor",
+          "Information Sciences Institute University of Southern California", "IAB and IESG",
+          "RARE WG-MSG Task Force 88", "KOI8-U Working Group", "The Internet Society",
+          "IAB Advisory Committee", "ISOC Board of Trustees", "Mitra", "RFC Editor, et al."
+          build_org(nil, name)
+        when "Internet Assigned Numbers Authority (IANA)"
+          build_org("IANA", "Internet Assigned Numbers Authority (IANA)")
+        when "ESnet Site Coordinating Comittee (ESCC)"
+          build_org("ESCC", "ESnet Site Coordinating Comittee (ESCC)")
+        when "Energy Sciences Network (ESnet)"
+          build_org("ESnet", "Energy Sciences Network (ESnet)")
+        when "International Telegraph and Telephone Consultative Committee of the International Telecommunication Union"
+          build_org("CCITT", name)
+        end
+      end
+
+      def build_org(abbr, name)
+        args = { name: [Bib::TypedLocalizedString.new(content: name, language: "en")] }
+        args[:abbreviation] = Bib::LocalizedString.new(content: abbr, language: "en") if abbr && !abbr.empty?
+        Bib::Organization.new(**args)
+      end
+
+      #
+      # Parse name, surname, and initials from full name
+      #
+      # @param [String] fname full name
+      # @param [String, nil] sname surname
+      # @param [String, nil] inits
+      #
+      # @return [Array<String, nil>] surname, initials, forename
+      #
+      def parse_surname_initials(fname, sname, inits) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+        regex = /(?:(?<name>\w{3,})\s)?(?<inits>(?:[A-Z]{1,2}(?:\.[\s-]?|\s))+)?/
+        match = fname&.match(regex)
+        surname = sname || fname&.sub(regex, "")&.strip
+        initials = inits || (match && match[:inits]&.strip)
+        [surname, initials, (match && match[:name])]
+      end
+
+      class FromRfcxml < Bib::Converter::BibXml::FromRfcxml # rubocop:disable Metrics/ClassLength
+        private
+
+        def pubid_type(id)
+          BibXMLParser.pubid_type(id)
+        end
+
+        def contributor # rubocop:disable Metrics/MethodLength
+          contribs = []
+          unless @reference.anchor&.match?(/^I-D/)
+            org = BibXMLParser.build_org("IETF", "Internet Engineering Task Force")
+            contribs << Bib::Contributor.new(
+              organization: org,
+              role: [Bib::Contributor::Role.new(type: "publisher")],
+            )
+          end
+          contribs + super
+        end
+
+        def person(author)
+          return unless author.fullname && author.fullname != "None"
+          return if BibXMLParser.full_name_org(author.fullname)
+
+          super
+        end
+
+        def organization(author)
+          if author.fullname
+            org = BibXMLParser.full_name_org(author.fullname)
+            return org if org
+          end
+          super
+        end
+
+        def person_name(author) # rubocop:disable Metrics/MethodLength
+          fname = author.fullname
+          surname, inits, forename = BibXMLParser.parse_surname_initials(
+            fname, author.surname, author.initials
+          )
+          fnames = []
+          if forename
+            fnames << Bib::FullNameType::Forename.new(
+              content: forename, language: "en", script: "Latn",
+            )
+          end
+          if inits
+            inits.split(/\.-?\s?|\s/).each do |i|
+              fnames << Bib::FullNameType::Forename.new(
+                initial: i, language: "en", script: "Latn",
+              )
+            end
+          end
+          Bib::FullName.new(
+            completename: Bib::LocalizedString.new(content: fname, language: "en"),
+            formatted_initials: inits ? Bib::LocalizedString.new(content: inits, language: "en") : nil,
+            surname: Bib::LocalizedString.new(content: surname, language: "en"),
+            forename: fnames,
+          )
+        end
+      end
+
+      class FromRfc < FromRfcxml # rubocop:disable Metrics/ClassLength
+        def transform # rubocop:disable Metrics/MethodLength
+          namespace::ItemData.new(
+            docnumber: doc_name&.sub(/^\w+\./, ""),
+            type: "standard",
+            docidentifier: docidentifiers,
+            status: status,
+            language: ["en"],
+            script: ["Latn"],
+            source: source,
+            title: title,
+            abstract: abstract,
+            contributor: contributor + workgroup_contributors,
+            date: date,
+            series: series,
+            keyword: keyword,
+            ext: ext,
+          )
+        end
+
+        private
+
+        def doc_name = @reference.doc_name
+
+        def docidentifiers
+          ids = []
+          if doc_name
+            ids << Bib::Docidentifier.new(type: "Internet-Draft", content: doc_name, primary: true)
+            ids << Bib::Docidentifier.new(type: "IETF", content: doc_name, scope: "docName")
+          end
+          ids + docid_from_series_info
+        end
+
+        def source = []
+
+        def contributor # rubocop:disable Metrics/MethodLength
+          contribs = []
+          org = BibXMLParser.build_org("IETF", "Internet Engineering Task Force")
+          contribs << Bib::Contributor.new(
+            organization: org,
+            role: [Bib::Contributor::Role.new(type: "publisher")],
+          )
+          contribs + Bib::Converter::BibXml::FromRfcxml.instance_method(:contributor).bind_call(self)
+        end
+
+        def doctype
+          namespace::Doctype.new(content: "rfc")
+        end
+
+        def series
+          front_si = @reference.front.series_info || []
+          front_si.filter_map do |si|
+            next if si.name == "DOI" || si.stream || si.status
+
+            t = Bib::Title.new(content: si.name, language: "en", script: "Latn")
+            Bib::Series.new(title: [t], number: si.value, type: "main")
+          end
+        end
+
+        def status
+          si = @reference.front.series_info&.find(&:status)
+          return unless si
+
+          stage = Bib::Status::Stage.new(content: si.status)
+          Bib::Status.new(stage: stage)
+        end
+
+        def versioned_internet_draft_id
+          internet_draft_series_info(@reference.front)
+        end
+
+        def docid_from_series_info
+          front_si = @reference.front.series_info || []
+          front_si.each_with_object([]) do |s, acc|
+            next unless s.name.casecmp("doi").zero?
+
+            acc << Bib::Docidentifier.new(type: "DOI", content: s.value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ietf/data_fetcher.rb

@@ -0,0 +1,278 @@
+require "etc"
+require "parallel"
+require "relaton/core"
+require_relative "../ietf"
+require_relative "bibxml_parser"
+require_relative "rfc/index"
+require_relative "rfc/entry"
+require_relative "wg_name_resolver"
+
+module Relaton
+  module Ietf
+    class DataFetcher < Core::DataFetcher
+      #
+      # Fetch documents
+      #
+      def fetch(source)
+        @source = source
+        case source
+        when "ietf-rfcsubseries" then fetch_ieft_rfcsubseries
+        when "ietf-internet-drafts" then fetch_ieft_internet_drafts
+        when "ietf-rfc-entries" then fetch_ieft_rfcs
+        end
+        index.save
+      end
+
+      private
+
+      def index
+        @index ||= Relaton::Index.find_or_create :IETF, file: "#{INDEXFILE}.yaml"
+      end
+
+      #
+      # Fetches ietf-rfcsubseries documents
+      #
+      def fetch_ieft_rfcsubseries
+        idx = Rfc::Index.from_xml(rfc_index)
+        rfc_map = (idx.rfc_entries || []).each_with_object({}) do |entry, h|
+          h[entry.doc_id] = entry
+        end
+        idx.subseries_entries.each do |entry|
+          save_doc entry.to_item(rfc_map, wg_names: wg_names)
+        end
+      end
+
+      #
+      # Fetches ietf-internet-drafts documents.
+      #
+      # Each work unit (one series, or one singleton XML) is processed
+      # end-to-end in a worker process: parse → link relations → serialize →
+      # write. Workers return Marshal-friendly index entries; the parent
+      # collects them and updates `Relaton::Index` and the duplicate-check set
+      # serially. Set `RELATON_IETF_PARALLEL_WORKERS=0` to force serial
+      # execution (useful for tests and debugging).
+      #
+      def fetch_ieft_internet_drafts
+        series_groups, singleton_paths = group_draft_paths
+
+        series_results = parallelize(series_groups.to_a) do |(series, paths_info)|
+          process_series(series, paths_info)
+        end.flatten(1)
+
+        singleton_results = parallelize(singleton_paths) do |path|
+          process_singleton(path)
+        end
+
+        (series_results + singleton_results).compact.each { |r| record_index_entry(r) }
+      end
+
+      #
+      # Run `block` once per item, in parallel worker processes when configured.
+      # `Parallel.map(items, in_processes: 0)` runs synchronously in the
+      # current process, which keeps tests deterministic and lets mocks work.
+      #
+      def parallelize(items, &block)
+        Parallel.map(items, in_processes: worker_count, &block)
+      end
+
+      def worker_count
+        ENV.fetch("RELATON_IETF_PARALLEL_WORKERS", Etc.nprocessors.to_s).to_i
+      end
+
+      #
+      # Filename-only scan: group versioned drafts by normalized series stem;
+      # everything else (non-versioned, non-`D.draft-`) goes to singletons.
+      # No XML parsing happens here — workers do that.
+      #
+      # @return [Array(Hash, Array<String>)]
+      #   series_groups: { normalized_series => [{path, ver, ref}, ...] }
+      #   singleton_paths: [path, ...]
+      #
+      def group_draft_paths
+        series_groups = {}
+        singleton_paths = []
+        Dir["bibxml-ids/*.xml"].each do |path|
+          file = File.basename(path, ".xml")
+          is_draft = file.include?("D.draft-")
+          ver = is_draft ? file[/(\d+)$/, 1] : nil
+          ref = file.sub(/^reference\.I-D\./, "").downcase
+          stem_match = is_draft && ver ? /^(draft-.+)-(\d{2})$/.match(ref) : nil
+          if stem_match
+            series = stem_match[1].gsub(/[.\s\/:-]+/, "-")
+            (series_groups[series] ||= []) << { path: path, ver: ver, ref: ref }
+          else
+            singleton_paths << path
+          end
+        end
+        [series_groups, singleton_paths]
+      end
+
+      #
+      # Worker: parse all files in a series, sort by version, append
+      # immediate-neighbor relations (skipped for bibxml), write each version
+      # and the un-versioned aggregator doc. Returns an array of index entries
+      # for the parent.
+      #
+      def process_series(series, paths_info)
+        sorted = paths_info.sort_by { |p| p[:ver].to_i }.map do |p|
+          bib = BibXMLParser.parse(File.read(p[:path], encoding: "UTF-8"))
+          bib.version = [Bib::Version.new(draft: p[:ver])]
+          p.merge(bib: bib, source: bib.source)
+        end
+        link_neighbor_relations(sorted) if @format != "bibxml"
+
+        results = sorted.map { |entry| serialize_and_write(entry[:bib]) }
+        results << serialize_and_write(build_unversioned_doc(series, sorted)) if @format != "bibxml"
+        results.compact
+      end
+
+      #
+      # Worker: parse + serialize + write a single non-grouped XML.
+      #
+      def process_singleton(path)
+        file = File.basename(path, ".xml")
+        is_draft = file.include?("D.draft-")
+        ver = is_draft ? file[/(\d+)$/, 1] : nil
+        bib = BibXMLParser.parse(File.read(path, encoding: "UTF-8"))
+        bib.version = [Bib::Version.new(draft: ver)] if ver
+        serialize_and_write(bib)
+      end
+
+      #
+      # Append immediate-neighbor `updates` / `updatedBy` relations in memory.
+      # Single-version series get no relations (no neighbors).
+      #
+      def link_neighbor_relations(sorted)
+        sorted.each_with_index do |entry, i|
+          if i.positive?
+            prev = sorted[i - 1]
+            entry[:bib].relation << version_relation({ ref: prev[:ref], source: prev[:source] }, "updates")
+          end
+          if i < sorted.size - 1
+            nxt = sorted[i + 1]
+            entry[:bib].relation << version_relation({ ref: nxt[:ref], source: nxt[:source] }, "updatedBy")
+          end
+        end
+      end
+
+      #
+      # Build (but do not write) the un-versioned series aggregator doc with
+      # `includes` relations to every version. Uses the latest version's
+      # title/abstract from memory.
+      #
+      # @return [Relaton::Ietf::ItemData, nil]
+      #
+      def build_unversioned_doc(series, sorted)
+        if sorted.empty?
+          Util.warn "No versions found for #{series}"
+          return nil
+        end
+
+        last_v = sorted.last[:bib]
+        docid = Bib::Docidentifier.new(type: "Internet-Draft", content: series, primary: true)
+        rel = sorted.map { |e| version_relation({ ref: e[:ref], source: e[:source] }, "includes") }
+        ItemData.new(
+          title: last_v.title, abstract: last_v.abstract, formattedref: Bib::Formattedref.new(content: series),
+          docidentifier: [docid], relation: rel
+        )
+      end
+
+      #
+      # Create bibitem relation
+      #
+      # @param [Hash] ver version reference, { ref:, source: }
+      # @param [String] type relation type
+      #
+      # @return [Relaton::Ietf::Relation] relation
+      #
+      def version_relation(ver, type)
+        docid = Bib::Docidentifier.new(type: "Internet-Draft", content: ver[:ref], primary: true)
+        bibitem = ItemData.new(formattedref: Bib::Formattedref.new(content: ver[:ref]), docidentifier: [docid], source: ver[:source])
+        Relaton::Ietf::Relation.new(type: type, bibitem: bibitem)
+      end
+
+      #
+      # Fetches ietf-rfc-entries documents
+      #
+      def fetch_ieft_rfcs
+        idx = Rfc::Index.from_xml(rfc_index)
+        idx.rfc_entries.each do |entry|
+          save_doc entry.to_item(nil, wg_names: wg_names)
+        rescue StandardError => e
+          Util.error "Error parsing #{entry.doc_id}: #{e.message}\n" \
+            "#{e.backtrace[0..5].join("\n")}"
+        end
+      end
+
+      #
+      # Get RFC index
+      #
+      # @return [Nokogiri::XML::Document] RFC index
+      #
+      def rfc_index
+        uri = URI "https://www.rfc-editor.org/rfc-index.xml"
+        Net::HTTP.get(uri)
+      end
+
+      def wg_names
+        @wg_names ||= WgNameResolver.fetch
+      end
+
+      #
+      # Save document to file (sequential path: serialize, write, index).
+      # Used by the rfcsubseries / rfc-entries fetchers; the I-D fetcher splits
+      # this into worker-safe `serialize_and_write` plus parent-only
+      # `record_index_entry` so the index is touched only in the main process.
+      #
+      # @param [Relaton::Ietf::Rfc::Entry, nil] entry
+      # @param [Boolean] check_duplicate check for duplicate
+      #
+      def save_doc(entry, check_duplicate: true)
+        result = serialize_and_write(entry)
+        record_index_entry(result, check_duplicate: check_duplicate) if result
+      end
+
+      #
+      # Worker-safe: serialize, compute output filename, write to disk, return
+      # a Marshal-friendly hash with the docid+file pair the parent needs to
+      # update `Relaton::Index` and `@files`. Does NOT touch instance state
+      # that has to stay consistent across workers (`@files`, the index).
+      #
+      # @param [#to_yaml, #to_xml, #to_rfcxml, nil] entry
+      # @return [Hash, nil]
+      #
+      def serialize_and_write(entry) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+        return nil unless entry
+
+        content = case @format
+                  when "xml" then entry.to_xml(bibdata: true)
+                  when "yaml" then entry.to_yaml
+                  when "bibxml" then entry.to_rfcxml
+                  else entry.send("to_#{@format}")
+                  end
+        id = if entry.respond_to?(:docidentifier)
+               entry.docidentifier.detect { |i| i.type == "Internet-Draft" && i.primary }&.content
+             end
+        id ||= entry.docnumber || entry.formattedref.content
+        file = output_file(id)
+        File.write file, content, encoding: "UTF-8"
+        primary = entry.docidentifier.detect(&:primary) || entry.docidentifier.first
+        { docnumber: entry.docnumber, file: file, index_id: primary.content }
+      end
+
+      #
+      # Parent-only: dedupe-check `@files` and update `Relaton::Index`. Called
+      # serially after workers return so index updates are race-free.
+      #
+      def record_index_entry(result, check_duplicate: true)
+        if check_duplicate && @files.include?(result[:file])
+          Util.warn "File #{result[:file]} already exists. Document: #{result[:docnumber]}"
+        elsif check_duplicate
+          @files << result[:file]
+        end
+        index.add_or_update result[:index_id], result[:file]
+      end
+
+    end
+  end
+end

data/lib/relaton/ietf/doctype.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Ietf
+    class Doctype < Bib::Doctype
+      TYPES = %w[rfc internet-draft].freeze
+
+      attribute :type, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/ietf/ext.rb

@@ -0,0 +1,63 @@
+require_relative "doctype"
+require_relative "processing_instructions"
+
+module Relaton
+  module Ietf
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :area, :string, collection: true, values: %W[
+        apt gen int ops rtg sec tsv Applications\sand\sReal-Time General
+        Internet Operations\sand\sManagement Routing Security Transport
+      ]
+      attribute :stream, :string, values: %w[IAB IETF Independent IRTF Legacy Editorial]
+      attribute :ipr, :string
+      attribute :pi, ProcessingInstructions
+      attribute :consensus, :string
+      attribute :index_include, :string
+      attribute :ipr_extract, :string
+      attribute :sort_refs, :string
+      attribute :sym_refs, :string
+      attribute :toc_include, :string
+      attribute :toc_depth, :string
+      attribute :show_on_front_page, :string
+
+      xml do
+        map_element "area", to: :area
+        map_element "stream", to: :stream
+        map_element "ipr", to: :ipr
+        map_element "pi", to: :pi
+        map_element "consensus", to: :consensus
+        map_element "indexInclude", to: :index_include
+        map_element "iprExtract", to: :ipr_extract
+        map_element "sortRefs", to: :sort_refs
+        map_element "symRefs", to: :sym_refs
+        map_element "tocInclude", to: :toc_include
+        map_element "tocDepth", to: :toc_depth
+        map_element "showOnFrontPage", to: :show_on_front_page
+      end
+
+      key_value do
+        map_element "schema_version", to: :schema_version, render_default: true
+        map_element "doctype", to: :doctype
+        map_element "subdoctype", to: :subdoctype
+        map_element "flavor", to: :flavor
+        map_element "ics", to: :ics
+        map_element "structuredidentifier", to: :structuredidentifier
+        map_element "area", to: :area
+        map_element "stream", to: :stream
+        map_element "ipr", to: :ipr
+        map_element "pi", to: :pi
+        map_element "consensus", to: :consensus
+        map_element "index_include", to: :index_include
+        map_element "ipr_extract", to: :ipr_extract
+        map_element "sort_refs", to: :sort_refs
+        map_element "sym_refs", to: :sym_refs
+        map_element "toc_include", to: :toc_include
+        map_element "toc_depth", to: :toc_depth
+        map_element "show_on_front_page", to: :show_on_front_page
+      end
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-ietf"]
+    end
+  end
+end

data/lib/relaton/ietf/item.rb

@@ -0,0 +1,16 @@
+require_relative "ext"
+
+module Relaton
+  module Ietf
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+    end
+  end
+end
+
+require_relative "relation"
+
+Relaton::Ietf::Item.attribute :relation, Relaton::Ietf::Relation,
+                              collection: true, initialize_empty: true

data/lib/relaton/ietf/item_base.rb

@@ -0,0 +1,18 @@
+module Relaton
+  module Ietf
+    class ItemBase < Lutaml::Model::Serializable
+      include Bib::NamespaceHelper
+
+      attr_accessor :type
+
+      model ItemData
+
+      instance_exec(&Bib::ItemShared::ATTRIBUTES)
+
+      xml do
+        map_attribute "type", to: :type
+        instance_exec(&Bib::ItemShared::XML_BODY)
+      end
+    end
+  end
+end

data/lib/relaton/ietf/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Ietf
+    class ItemData < Bib::ItemData
+    end
+  end
+end