relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/calconnect/hit_collection.rb

@@ -0,0 +1,92 @@
+module Relaton::Calconnect
+  class HitCollection < Relaton::Core::HitCollection
+    GHURL = "https://raw.githubusercontent.com/relaton/relaton-data-calconnect/refs/heads/v2/".freeze
+
+    # @param ref [Strig]
+    # @param year [String]
+    def initialize(ref, year = nil)
+      super
+      @array = search_index(ref).map { |row| Hit.new(row, self) }
+    end
+
+    private
+
+    # @return [Relaton::Index::Type]
+    def index
+      Relaton::Index.find_or_create :CC, url: "#{GHURL}#{INDEXFILE}.zip",
+                                         file: "#{INDEXFILE}.yaml",
+                                         pubid_class: ::Pubid::Calconnect::Identifier
+    end
+
+    #
+    # The index rows matching a reference, most recent first.
+    #
+    # **The pubid is passed to `Index::Type#search`, not the string.**
+    # `search_candidates` narrows only when the argument is not a `String`, and
+    # a block alone never narrows — so the plain string this used to pass
+    # disabled the binary search however the index was built. `pubid_class:` on
+    # the index alone fixes nothing; both had to change together.
+    #
+    # This also ends the substring scan the old string search did, which was
+    # silently ambiguous: `CC/DIR 1000` answered with all five `CC/DIR 1000x`
+    # documents and `CC/A 1` with every `CC/A 1xxx`. A number now matches
+    # exactly, and a leading zero is significant (`CC/A 0001` is not `CC/A 1`).
+    #
+    # The rows are selected with pubid's subset match `pubid === row`, the
+    # default of `Index::Type#search`. A date that the reference omits matches
+    # any value. pubid declares `series` strict for CalConnect, so the series
+    # keeps `CC/CD 51016` and `CC/WD 51016` apart, and a series-less
+    # `CC 36010` does not match `CC/WD 36010`.
+    #
+    # @param ref [String]
+    # @return [Array<Hash>] matching index rows
+    #
+    def search_index(ref)
+      pubid = parse_ref ref
+      return [] unless pubid
+
+      index.search(pubid).sort_by { |row| [recency_key(row[:id]), row[:file]] }
+    end
+
+    #
+    # Parse a user reference into a `Pubid::Calconnect::Identifier`, or nil.
+    #
+    # A reference pubid rejects is a **miss, not an error**: this returns nil
+    # outside the transport rescue in `Bibliography.search`, so it never becomes
+    # a `Relaton::RequestError`.
+    #
+    # @param ref [String]
+    # @return [Pubid::Calconnect::Identifier, nil]
+    #
+    # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
+    # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
+    # `"..." is not a recognized standards identifier`
+    # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`), and `Db#fetch`
+    # logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
+    # Rescuing here would collapse "this identifier is malformed" into "no
+    # such document", leaving a caller unable to tell them apart.
+    def parse_ref(ref)
+      ::Pubid::Calconnect::Identifier.parse ref.to_s.strip
+    end
+
+    #
+    # Sort key placing the most recent document first.
+    #
+    # The index is sorted by number, so rows sharing a number arrive in no
+    # meaningful order — without this, `Bibliography.get "CC/S 0601"` would
+    # answer with an arbitrary one of the 2005 and 2006 documents. Segments are
+    # compared as integers and negated for descending order; an absent month or
+    # day sorts as 0, which is right because only one row in the corpus carries
+    # either.
+    #
+    # @param id [Pubid::Calconnect::Identifier]
+    # @return [Array<Integer>]
+    #
+    def recency_key(id)
+      date = id.date
+      return [0, 0, 0] unless date
+
+      [date.year, date.month, date.day].map { |part| -part.to_i }
+    end
+  end
+end

data/lib/relaton/calconnect/item_data.rb

@@ -0,0 +1,12 @@
+module Relaton
+  module Calconnect
+    class ItemData < Relaton::Bib::ItemData
+      def create_id(without_date: false)
+        docid = docidentifier.find(&:primary) || docidentifier.first
+        return unless docid
+
+        self.id = docid.content.gsub(/\W+/, "")
+      end
+    end
+  end
+end

data/lib/relaton/calconnect/model/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Calconnect
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/calconnect/model/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Calconnect
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/calconnect/model/doctype.rb

@@ -0,0 +1,11 @@
+module Relaton
+  module Calconnect
+    class Doctype < Bib::Doctype
+      TYPES = %W[
+        directive guide specification standard report administrative amendment technical\scorrigendum advisory
+      ].freeze
+
+      attribute :type, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/calconnect/model/ext.rb

@@ -0,0 +1,11 @@
+require_relative "doctype"
+
+module Relaton
+  module Calconnect
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-cc"]
+    end
+  end
+end

data/lib/relaton/calconnect/model/item.rb

@@ -0,0 +1,20 @@
+require "relaton/bib"
+require_relative "../item_data"
+require_relative "../docidentifier"
+require_relative "ext"
+
+module Relaton
+  module Calconnect
+    class Item < Bib::Item
+      model ItemData
+
+      # The flavor's own Docidentifier, so every parsed record carries a
+      # `#pubid` for `DataFetcher#index_id` to key the index-v2 on. Bibitem and
+      # Bibdata subclass Item, so they inherit it.
+      attribute :docidentifier, Docidentifier, collection: true,
+                                               initialize_empty: true
+
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/calconnect/processor.rb

@@ -0,0 +1,72 @@
+require "relaton/core/processor"
+
+module Relaton::Calconnect
+  class Processor < Relaton::Core::Processor
+    attr_reader :idtype
+
+    def initialize # rubocop:disable Lint/MissingSuper
+      @short = :relaton_calconnect
+      @prefix = "CC"
+      @defaultprefix = %r{^CC(?!\w)}
+      @idtype = "CC"
+      @datasets = %w[calconnect-org]
+      @pubid_flavor = :Calconnect
+    end
+
+    # @param code [String]
+    # @param date [String, nil] year
+    # @param opts [Hash]
+    # @return [Relaton::Calconnect::ItemData, nil]
+    def get(code, date, opts)
+      require_relative "../calconnect"
+      Bibliography.get(code, date, opts)
+    end
+
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      require_relative "data_fetcher"
+      DataFetcher.fetch(**opts)
+    end
+
+    # @param xml [String]
+    # @return [Relaton::Calconnect::ItemData]
+    def from_xml(xml)
+      require_relative "../calconnect"
+      Item.from_xml xml
+    end
+
+    # @param hash [Hash]
+    # @return [Relaton::Calconnect::ItemData]
+    def hash_to_bib(hash)
+      require_relative "../calconnect"
+      Item.from_yaml hash.to_yaml
+    end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def grammar_hash
+      require_relative "../calconnect"
+      @grammar_hash ||= ::Relaton::Calconnect.grammar_hash
+    end
+
+    #
+    # Remove index file
+    #
+    # `url: true` names the cached file. No `pubid_class:`: `Type#remove_file`
+    # deletes the file and never reads the index.
+    #
+    def remove_index_file
+      require_relative "../calconnect"
+      Relaton::Index.find_or_create(
+        :CC, url: true, file: "#{INDEXFILE}.yaml"
+      ).remove_file
+    end
+  end
+end

data/lib/relaton/calconnect/scraper.rb

@@ -0,0 +1,87 @@
+require "mechanize"
+require "stringio"
+require "zip"
+require_relative "model/item"
+require_relative "model/bibdata"
+
+module Relaton
+  module Calconnect
+    class Scraper
+      include Core::HashKeysSymbolizer
+      include Core::ArrayWrapper
+
+      RELEASE_ASSET_URL = "https://github.com/%<owner>s/%<repo>s/releases/download/" \
+                          "%<tag>s/%<asset_stem>s.zip".freeze
+
+      # @param errors [Hash] error tracking hash
+      def initialize(errors = {})
+        @errors = errors
+      end
+
+      #
+      # Parse an aggregate-index document entry: download the per-document
+      # GitHub release zip, extract the RXL, and parse it into a bibitem.
+      #
+      # @param hit [Hash] document entry from /cc/index.json
+      #
+      # @return [Relaton::Calconnect::ItemData] bibliographic item
+      #
+      def parse_page(hit)
+        zip_data = download_release_zip hit
+        rxl = extract_rxl zip_data, rxl_filename(hit)
+        xml = normalize_rxl rxl
+        Item.from_xml xml
+      end
+
+      private
+
+      def release_zip_url(hit)
+        source = hit["source"] || {}
+        format(
+          RELEASE_ASSET_URL,
+          owner: source["owner"],
+          repo: source["repo"],
+          tag: source["tag"],
+          asset_stem: asset_stem(hit),
+        )
+      end
+
+      def rxl_filename(hit)
+        "#{asset_stem(hit)}.rxl"
+      end
+
+      # The release asset uses the tag with the slash replaced by a hyphen,
+      # which encodes both the document id and the release qualifier
+      # (e.g. `ed1`, `ed1-wd`).
+      def asset_stem(hit)
+        (hit["source"] && hit["source"]["tag"] || "").tr("/", "-")
+      end
+
+      def download_release_zip(hit)
+        url = release_zip_url(hit)
+        agent.get(url).body
+      rescue Mechanize::ResponseCodeError => e
+        raise "Failed to download release zip #{url}: HTTP #{e.response_code}"
+      end
+
+      def agent
+        @agent ||= Mechanize.new
+      end
+
+      def extract_rxl(zip_data, filename)
+        Zip::File.open_buffer(StringIO.new(zip_data)) do |zip|
+          entry = zip.find_entry(filename)
+          raise "RXL file #{filename} not found in release zip" unless entry
+
+          return entry.get_input_stream.read
+        end
+      end
+
+      def normalize_rxl(xml)
+        xml.gsub(%r{(</?)technical-committee(>)}, '\1committee\2')
+          .gsub(%r{type="(?:csd|CC)"(?=>)}i, '\0 primary="true"')
+          .gsub(%r{type="Technical committee"}, 'type="technical-committee"')
+      end
+    end
+  end
+end

data/lib/relaton/calconnect/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Calconnect
+    module Util
+      extend Bib::Util
+      PROGNAME = "relaton-calconnect".freeze
+    end
+  end
+end

data/lib/relaton/calconnect.rb

@@ -0,0 +1,36 @@
+# Not lazy: DataFetcher names ::Pubid::Calconnect::Identifier as the index
+# `pubid_class:`, and Docidentifier parses every docid through it.
+# (The ECMA/IANA/IHO/OGC form; spec/relaton/lazy_loading_spec.rb guards that
+# this file is not itself loaded when a Db is built.)
+require "pubid"
+require "relaton/index"
+require "relaton/core"
+require_relative "version"
+require_relative "calconnect/model/item"
+require_relative "calconnect/util"
+require_relative "calconnect/model/bibitem"
+require_relative "calconnect/model/bibdata"
+require_relative "calconnect/bibliography"
+require_relative "calconnect/hit_collection"
+require_relative "calconnect/hit"
+require_relative "calconnect/scraper"
+
+module Relaton
+  module Calconnect
+    # The one index this flavor builds and reads: pubid-keyed rows
+    # (`_type: pubid:calconnect:standard`), via
+    # `pubid_class: ::Pubid::Calconnect::Identifier`.
+    # `relaton-data-calconnect`'s crawler derives the legacy `index-v1` from
+    # these rows for released consumers, so it is not produced or read here.
+    INDEXFILE = "index-v2".freeze
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/ccsds/bibliography.rb

@@ -0,0 +1,63 @@
+require_relative "hit_collection"
+
+module Relaton
+  module Ccsds
+    module Bibliography
+      extend self
+
+      #
+      # Search for CCSDS standards by document reference.
+      #
+      # @param [String] ref document reference
+      #
+      # @return [RelatonCcsds::HitCollection] collection of hits
+      #
+      def search(ref)
+        HitCollection.new(ref).fetch
+      end
+
+      #
+      # Get CCSDS standard by document reference.
+      # If format is not specified, then all format will be returned.
+      #
+      # @param reference [String]
+      # @param year [String, nil]
+      # @param opts [Hash]
+      # @option opts [String] :format format of fetched document (DOC, PDF)
+      #
+      # @return [RelatonCcsds::BibliographicItem]
+      #
+      def get(reference, _year = nil, opts = {})
+        ref, opts = parse_format(reference, opts)
+        Util.info "Fetching from Relaton repository ...", key: reference
+        item, hit = fetch_item(ref)
+        if item.nil? || filter_sources(item, opts[:format])
+          Util.info "Not found.", key: reference
+          return nil
+        end
+        Util.info "Found: `#{hit[:code]}`.", key: reference
+        item
+      end
+
+      private
+
+      def parse_format(reference, opts)
+        ref = reference.sub(/\s\((DOC|PDF)\)$/, "")
+        opts[:format] ||= Regexp.last_match(1)
+        [ref, opts]
+      end
+
+      def fetch_item(ref)
+        hit = search(ref).first
+        [hit&.item, hit&.hit]
+      end
+
+      def filter_sources(item, format)
+        return unless format
+
+        item.source = item.source.select { |s| s.type == format.downcase }
+        item.source.empty?
+      end
+    end
+  end
+end

data/lib/relaton/ccsds/data/fetcher.rb

@@ -0,0 +1,252 @@
+require "json"
+require "mechanize"
+require "relaton/index"
+require "pubid"
+require_relative "../../ccsds"
+require_relative "parser"
+
+module Relaton
+  module Ccsds
+    class DataFetcher < Relaton::Core::DataFetcher
+      TRRGX = /\s-\s\w+\sTranslated$/
+
+      def agent
+        return @agent if @agent
+
+        @agent = Mechanize.new
+        @agent.request_headers = { "Accept" => "application/json;odata=verbose" }
+        @agent
+      end
+
+      # Pubid index (index-v2): `:id` is the lean pubid hash. index-v1 (the
+      # pubid-v1 hash index for the released gem line) is rebuilt separately by
+      # the data repo's build_index_v1.rb, in its own process with a pubid-v1
+      # bundle, because pubid v1 and v2 both define Pubid::Ccsds::Identifier and
+      # cannot coexist here.
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :ccsds, file: "#{INDEXFILE}.yaml", pubid_class: Pubid::Ccsds::Identifier
+        )
+      end
+
+      def fetch(_source = nil)
+        fetch_docs "https://ccsds.org/publications/ccsdsallpubs/"
+        index.save
+      end
+
+      #
+      # Fetch documents from url
+      #
+      # @param [String] url
+      #
+      # @return [void]
+      #
+      def fetch_docs(url)
+        resp = agent.get(url)
+        json = JSON.parse resp.body.match(/const config = (.*);/)[1]
+        @array = json["data"].map { |doc| parse_and_save doc, json["data"] }
+      end
+
+      #
+      # Parse document and save to file
+      #
+      # @param [Hash] doc document data
+      # @param [Array<Array<String>>] data collection of documents
+      #   0 - empty
+      #   1 - center/a HTML element with href to PDF
+      #   2 - a HTML element with href to HTML and document ID content (e.g. "CCSDS 123.0-B-1")
+      #   3 - document title
+      #   4 - document series (e.g. "Blue Book", "Silver Book", etc)
+      #   5 - issue number
+      #   6 - publication date (e.g. "August 2020")
+      #   7 - abstract
+      #   8 - Working Group as `{WG name} <a href="{path}" ...`
+      #   9 - ISO Equivalent as `{ISO id} <a href="{uri}" ...`
+      #  10 - Patent Licensing. Some docs has this field. Content is same and looks not useful.
+      #  11 - Extra Information. Looks not useful.
+      #
+      # @return [void]
+      #
+      def parse_and_save(doc, data)
+        bibitem = DataParser.new(doc, data).parse
+        if doc[4] == "Silver Book"
+          predecessor = DataParser.new(doc, data, bibitem).parse
+          save_bib predecessor
+        end
+        save_bib bibitem
+      end
+
+      #
+      # Save bibitem to file
+      #
+      # @param [Relaton::Ccsds::Item] bib bibitem
+      #
+      # @return [void]
+      #
+      def save_bib(bib) # rubocop:disable Metrics/AbcSize
+        search_instance_translation bib
+        file = output_file(bib.docidentifier.first.content)
+        merge_links bib, file
+        File.write file, serialize(bib), encoding: "UTF-8"
+        index.add_or_update Pubid::Ccsds::Identifier.parse(bib.docidentifier.first.content), file
+      rescue StandardError => e
+        puts "Failed to save #{bib.docidentifier.first.content}: #{e.message}\n#{e.backtrace[0..5].join("\n")}"
+      end
+
+      #
+      # Search translation and instance relation
+      #
+      # @param [Relaton::Ccsds::Item] bib translation bibitem
+      #
+      # @return [void]
+      #
+      def search_instance_translation(bib)
+        pubid = bib.docidentifier.first.pubid
+        return unless pubid # unparseable content: no relation to search
+
+        bibid = pubid.exclude(:language)
+        if bibid == pubid
+          search_translations bibid, bib # no language: this is an instance
+        else
+          search_relations bibid, bib # has a language: this is a translation
+        end
+      end
+
+      #
+      # Search instance or translation relation
+      #
+      # @param [Pubid::Ccsds::Identifier] bibid_pid language-less instance id
+      # @param [Relaton::Ccsds::ItemData] bib instance or translation bibitem
+      #
+      # @return [void]
+      #
+      def search_relations(bibid_pid, bib)
+        # search(bibid_pid) narrows candidates by number via binary search first.
+        index.search(bibid_pid) do |row|
+          # Match language-agnostically: `bibid_pid` is language-less, so a
+          # translated row and the instance both match. `===` cannot do this —
+          # pubid declares CCSDS `language` `subset_strict`, so a nil reference
+          # language means "has none" instead of "any". Mirrors `HitCollection`.
+          next unless row[:id].exclude(:language) == bibid_pid
+          next if row[:id] == bib.docidentifier.first.pubid # exclude the document's own row
+
+          create_relations bib, row[:file]
+        end
+      end
+
+      def search_translations(bibid_pid, bib)
+        # will call create_instance_relation if
+        # there are same identifiers in index but with word "Translated"
+        # search(bibid_pid) narrows candidates by number via binary search first.
+        index.search(bibid_pid) do |row|
+          # Only translated rows (`row[:id].language`), matched language-agnostically.
+          next unless row[:id].language && row[:id].exclude(:language) == bibid_pid
+
+          create_instance_relation bib, row[:file]
+        end
+      end
+
+      #
+      # Create translation or instance relation and save to file
+      #
+      # @param [Relaton::Ccsds::ItemData] bib bibliographic item
+      # @param [String] file translation or instance file
+      #
+      # @return [void]
+      #
+      def create_relations(bib, file)
+        inst = parse_file file
+        type1, type2 = translation_relation_types(inst)
+        create_relation(inst, type1) { |rel| bib.relation << rel }
+        create_relation(bib, type2) { |rel| inst.relation << rel }
+        File.write file, serialize(inst), encoding: "UTF-8"
+      end
+
+      def parse_file(file)
+        case @format
+        when "yaml" then Item.from_yaml File.read(file, encoding: "UTF-8")
+        when "xml" then Item.from_xml File.read(file, encoding: "UTF-8")
+        else
+          raise "Unknown format #{@format}"
+        end
+      end
+
+      #
+      # Translation or instance relation types
+      #
+      # @param [Relaton::Ccsds::ItemData] bib bibliographic item
+      #
+      # @return [Array<String>] relation types
+      #
+      def translation_relation_types(bib)
+        if bib.docidentifier.first.content.match?(TRRGX)
+          ["hasTranslation"] * 2
+        else
+          ["instanceOf", "hasInstance"]
+        end
+      end
+
+      #
+      # Create instance relation and save to file
+      #
+      # @param [Relaton::Ccsds::Item] bib bibliographic item
+      # @param [String] file file name
+      #
+      # @return [void]
+      #
+      def create_instance_relation(bib, file)
+        inst = parse_file file
+        create_relation(inst, "hasInstance") { |rel| bib.relation << rel }
+        create_relation(bib, "instanceOf") { |rel| inst.relation << rel }
+        File.write file, serialize(inst), encoding: "UTF-8"
+      end
+
+      #
+      # Create relation
+      #
+      # @param [Relaton::Ccsds::Item] bib the related bibliographic item
+      # @param [String] type type of relation
+      #
+      # @return [Relaton::Bib::Relation] relation
+      #
+      def create_relation(bib, type)
+        bib_docid = bib.docidentifier.first
+        return unless bib_docid
+
+        docid = Bib::Docidentifier.from_yaml(bib_docid.to_yaml)
+        rel = Relaton::Bib::ItemData.new docidentifier: [docid], formattedref: Relaton::Bib::Formattedref.new(content: bib_docid.content.dup)
+        yield Relaton::Bib::Relation.new(type: type, bibitem: rel)
+      end
+
+      #
+      # Merge identical documents with different links (updaes given bibitem)
+      #
+      # @param [Relaton::Ccsds::Item] bib bibliographic item
+      # @param [String] file path to existing document
+      #
+      # @return [void]
+      #
+      def merge_links(bib, file) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        # skip merging when new file
+        unless @files.include?(file)
+          @files << file
+          return
+        end
+
+        puts "(#{file}) file already exists. Trying to merge links ..."
+
+        bib2 = parse_file file
+        bib2.source.each do |src|
+          next if bib.source.any? { |s| s.type == src.type }
+
+          bib.source << src
+        end
+        Util.info "links are merged.", key: file
+      end
+
+      def to_yaml(bib) = bib.to_yaml
+      def to_xml(bib) = bib.to_xml(bibdata: true)
+      def to_bibxml(bib) = bib.to_rfcxml
+    end
+  end
+end