relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/calconnect/processor.rb

@@ -0,0 +1,66 @@
+require "relaton/core/processor"
+
+module Relaton::Calconnect
+  class Processor < Relaton::Core::Processor
+    attr_reader :idtype
+
+    def initialize # rubocop:disable Lint/MissingSuper
+      @short = :relaton_calconnect
+      @prefix = "CC"
+      @defaultprefix = %r{^CC(?!\w)}
+      @idtype = "CC"
+      @datasets = %w[calconnect-org]
+    end
+
+    # @param code [String]
+    # @param date [String, nil] year
+    # @param opts [Hash]
+    # @return [Relaton::Calconnect::ItemData, nil]
+    def get(code, date, opts)
+      require_relative "../calconnect"
+      Bibliography.get(code, date, opts)
+    end
+
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      require_relative "data_fetcher"
+      DataFetcher.fetch(**opts)
+    end
+
+    # @param xml [String]
+    # @return [Relaton::Calconnect::ItemData]
+    def from_xml(xml)
+      require_relative "../calconnect"
+      Item.from_xml xml
+    end
+
+    # @param hash [Hash]
+    # @return [Relaton::Calconnect::ItemData]
+    def hash_to_bib(hash)
+      require_relative "../calconnect"
+      Item.from_yaml hash.to_yaml
+    end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def grammar_hash
+      require_relative "../calconnect"
+      @grammar_hash ||= ::Relaton::Calconnect.grammar_hash
+    end
+
+    #
+    # Remove index file
+    #
+    def remove_index_file
+      require_relative "../calconnect"
+      Relaton::Index.find_or_create(:CC, url: true, file: "#{INDEXFILE}.yaml").remove_file
+    end
+  end
+end

data/lib/relaton/calconnect/scraper.rb

@@ -0,0 +1,87 @@
+require "mechanize"
+require "stringio"
+require "zip"
+require_relative "model/item"
+require_relative "model/bibdata"
+
+module Relaton
+  module Calconnect
+    class Scraper
+      include Core::HashKeysSymbolizer
+      include Core::ArrayWrapper
+
+      RELEASE_ASSET_URL = "https://github.com/%<owner>s/%<repo>s/releases/download/" \
+                          "%<tag>s/%<asset_stem>s.zip".freeze
+
+      # @param errors [Hash] error tracking hash
+      def initialize(errors = {})
+        @errors = errors
+      end
+
+      #
+      # Parse an aggregate-index document entry: download the per-document
+      # GitHub release zip, extract the RXL, and parse it into a bibitem.
+      #
+      # @param hit [Hash] document entry from /cc/index.json
+      #
+      # @return [Relaton::Calconnect::ItemData] bibliographic item
+      #
+      def parse_page(hit)
+        zip_data = download_release_zip hit
+        rxl = extract_rxl zip_data, rxl_filename(hit)
+        xml = normalize_rxl rxl
+        Item.from_xml xml
+      end
+
+      private
+
+      def release_zip_url(hit)
+        source = hit["source"] || {}
+        format(
+          RELEASE_ASSET_URL,
+          owner: source["owner"],
+          repo: source["repo"],
+          tag: source["tag"],
+          asset_stem: asset_stem(hit),
+        )
+      end
+
+      def rxl_filename(hit)
+        "#{asset_stem(hit)}.rxl"
+      end
+
+      # The release asset uses the tag with the slash replaced by a hyphen,
+      # which encodes both the document id and the release qualifier
+      # (e.g. `ed1`, `ed1-wd`).
+      def asset_stem(hit)
+        (hit["source"] && hit["source"]["tag"] || "").tr("/", "-")
+      end
+
+      def download_release_zip(hit)
+        url = release_zip_url(hit)
+        agent.get(url).body
+      rescue Mechanize::ResponseCodeError => e
+        raise "Failed to download release zip #{url}: HTTP #{e.response_code}"
+      end
+
+      def agent
+        @agent ||= Mechanize.new
+      end
+
+      def extract_rxl(zip_data, filename)
+        Zip::File.open_buffer(StringIO.new(zip_data)) do |zip|
+          entry = zip.find_entry(filename)
+          raise "RXL file #{filename} not found in release zip" unless entry
+
+          return entry.get_input_stream.read
+        end
+      end
+
+      def normalize_rxl(xml)
+        xml.gsub(%r{(</?)technical-committee(>)}, '\1committee\2')
+          .gsub(%r{type="(?:csd|CC)"(?=>)}i, '\0 primary="true"')
+          .gsub(%r{type="Technical committee"}, 'type="technical-committee"')
+      end
+    end
+  end
+end

data/lib/relaton/calconnect/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Calconnect
+    module Util
+      extend Bib::Util
+      PROGNAME = "relaton-calconnect".freeze
+    end
+  end
+end

data/lib/relaton/calconnect.rb

@@ -0,0 +1,26 @@
+require "relaton/index"
+require "relaton/core"
+require_relative "version"
+require_relative "calconnect/model/item"
+require_relative "calconnect/util"
+require_relative "calconnect/model/bibitem"
+require_relative "calconnect/model/bibdata"
+require_relative "calconnect/bibliography"
+require_relative "calconnect/hit_collection"
+require_relative "calconnect/hit"
+require_relative "calconnect/scraper"
+
+module Relaton
+  module Calconnect
+    INDEXFILE = "index-v1".freeze
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/ccsds/bibliography.rb

@@ -0,0 +1,63 @@
+require_relative "hit_collection"
+
+module Relaton
+  module Ccsds
+    module Bibliography
+      extend self
+
+      #
+      # Search for CCSDS standards by document reference.
+      #
+      # @param [String] ref document reference
+      #
+      # @return [RelatonCcsds::HitCollection] collection of hits
+      #
+      def search(ref)
+        HitCollection.new(ref).fetch
+      end
+
+      #
+      # Get CCSDS standard by document reference.
+      # If format is not specified, then all format will be returned.
+      #
+      # @param reference [String]
+      # @param year [String, nil]
+      # @param opts [Hash]
+      # @option opts [String] :format format of fetched document (DOC, PDF)
+      #
+      # @return [RelatonCcsds::BibliographicItem]
+      #
+      def get(reference, _year = nil, opts = {})
+        ref, opts = parse_format(reference, opts)
+        Util.info "Fetching from Relaton repository ...", key: reference
+        item, hit = fetch_item(ref)
+        if item.nil? || filter_sources(item, opts[:format])
+          Util.info "Not found.", key: reference
+          return nil
+        end
+        Util.info "Found: `#{hit[:code]}`.", key: reference
+        item
+      end
+
+      private
+
+      def parse_format(reference, opts)
+        ref = reference.sub(/\s\((DOC|PDF)\)$/, "")
+        opts[:format] ||= Regexp.last_match(1)
+        [ref, opts]
+      end
+
+      def fetch_item(ref)
+        hit = search(ref).first
+        [hit&.item, hit&.hit]
+      end
+
+      def filter_sources(item, format)
+        return unless format
+
+        item.source = item.source.select { |s| s.type == format.downcase }
+        item.source.empty?
+      end
+    end
+  end
+end

data/lib/relaton/ccsds/data/fetcher.rb

@@ -0,0 +1,247 @@
+require "json"
+require "mechanize"
+require "relaton/index"
+require "pubid"
+require_relative "../../ccsds"
+require_relative "parser"
+
+module Relaton
+  module Ccsds
+    class DataFetcher < Relaton::Core::DataFetcher
+      TRRGX = /\s-\s\w+\sTranslated$/
+
+      def agent
+        return @agent if @agent
+
+        @agent = Mechanize.new
+        @agent.request_headers = { "Accept" => "application/json;odata=verbose" }
+        @agent
+      end
+
+      # Pubid index (index-v2): `:id` is the lean pubid hash. index-v1 (the
+      # pubid-v1 hash index for the released gem line) is rebuilt separately by
+      # the data repo's build_index_v1.rb, in its own process with a pubid-v1
+      # bundle, because pubid v1 and v2 both define Pubid::Ccsds::Identifier and
+      # cannot coexist here.
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :ccsds, file: "#{INDEXFILE}.yaml", pubid_class: Pubid::Ccsds::Identifier
+        )
+      end
+
+      def fetch(_source = nil)
+        fetch_docs "https://ccsds.org/publications/ccsdsallpubs/"
+        index.save
+      end
+
+      #
+      # Fetch documents from url
+      #
+      # @param [String] url
+      #
+      # @return [void]
+      #
+      def fetch_docs(url)
+        resp = agent.get(url)
+        json = JSON.parse resp.body.match(/const config = (.*);/)[1]
+        @array = json["data"].map { |doc| parse_and_save doc, json["data"] }
+      end
+
+      #
+      # Parse document and save to file
+      #
+      # @param [Hash] doc document data
+      # @param [Array<Array<String>>] data collection of documents
+      #   0 - empty
+      #   1 - center/a HTML element with href to PDF
+      #   2 - a HTML element with href to HTML and document ID content (e.g. "CCSDS 123.0-B-1")
+      #   3 - document title
+      #   4 - document series (e.g. "Blue Book", "Silver Book", etc)
+      #   5 - issue number
+      #   6 - publication date (e.g. "August 2020")
+      #   7 - abstract
+      #   8 - Working Group as `{WG name} <a href="{path}" ...`
+      #   9 - ISO Equivalent as `{ISO id} <a href="{uri}" ...`
+      #  10 - Patent Licensing. Some docs has this field. Content is same and looks not useful.
+      #  11 - Extra Information. Looks not useful.
+      #
+      # @return [void]
+      #
+      def parse_and_save(doc, data)
+        bibitem = DataParser.new(doc, data).parse
+        if doc[4] == "Silver Book"
+          predecessor = DataParser.new(doc, data, bibitem).parse
+          save_bib predecessor
+        end
+        save_bib bibitem
+      end
+
+      #
+      # Save bibitem to file
+      #
+      # @param [Relaton::Ccsds::Item] bib bibitem
+      #
+      # @return [void]
+      #
+      def save_bib(bib) # rubocop:disable Metrics/AbcSize
+        search_instance_translation bib
+        file = output_file(bib.docidentifier.first.content)
+        merge_links bib, file
+        File.write file, serialize(bib), encoding: "UTF-8"
+        index.add_or_update Pubid::Ccsds::Identifier.parse(bib.docidentifier.first.content), file
+      rescue StandardError => e
+        puts "Failed to save #{bib.docidentifier.first.content}: #{e.message}\n#{e.backtrace[0..5].join("\n")}"
+      end
+
+      #
+      # Search translation and instance relation
+      #
+      # @param [Relaton::Ccsds::Item] bib translation bibitem
+      #
+      # @return [void]
+      #
+      def search_instance_translation(bib)
+        bibid = bib.docidentifier.first.content.dup
+        if bibid.sub!(TRRGX, "")
+          search_relations bibid, bib
+        else
+          search_translations bibid, bib
+        end
+      end
+
+      #
+      # Search instance or translation relation
+      #
+      # @param [String] bibid instance bibitem id
+      # @param [Relaton::Ccsds::ItemData] bib instance or translation bibitem
+      #
+      # @return [void]
+      #
+      def search_relations(bibid, bib)
+        bibid_pid = ::Pubid::Ccsds::Identifier.parse(bibid)
+        # search(bibid_pid) narrows candidates by number via binary search first.
+        index.search(bibid_pid) do |row|
+          id = row[:id].exclude(:language)
+          # TODO: smiplify this line?
+          next if id != bibid_pid || row[:id] == bib.docidentifier.first.content
+
+          create_relations bib, row[:file]
+        end
+      end
+
+      def search_translations(bibid, bib)
+        bibid_pid = ::Pubid::Ccsds::Identifier.parse(bibid)
+        # will call create_instance_relation if
+        # there are same identifiers in index but with word "Translated"
+        # search(bibid_pid) narrows candidates by number via binary search first.
+        index.search(bibid_pid) do |row|
+          next unless row[:id].language && row[:id].exclude(:language) == bibid_pid
+
+          create_instance_relation bib, row[:file]
+        end
+      end
+
+      #
+      # Create translation or instance relation and save to file
+      #
+      # @param [Relaton::Ccsds::ItemData] bib bibliographic item
+      # @param [String] file translation or instance file
+      #
+      # @return [void]
+      #
+      def create_relations(bib, file)
+        inst = parse_file file
+        type1, type2 = translation_relation_types(inst)
+        create_relation(inst, type1) { |rel| bib.relation << rel }
+        create_relation(bib, type2) { |rel| inst.relation << rel }
+        File.write file, serialize(inst), encoding: "UTF-8"
+      end
+
+      def parse_file(file)
+        case @format
+        when "yaml" then Item.from_yaml File.read(file, encoding: "UTF-8")
+        when "xml" then Item.from_xml File.read(file, encoding: "UTF-8")
+        else
+          raise "Unknown format #{@format}"
+        end
+      end
+
+      #
+      # Translation or instance relation types
+      #
+      # @param [Relaton::Ccsds::ItemData] bib bibliographic item
+      #
+      # @return [Array<String>] relation types
+      #
+      def translation_relation_types(bib)
+        if bib.docidentifier.first.content.match?(TRRGX)
+          ["hasTranslation"] * 2
+        else
+          ["instanceOf", "hasInstance"]
+        end
+      end
+
+      #
+      # Create instance relation and save to file
+      #
+      # @param [Relaton::Ccsds::Item] bib bibliographic item
+      # @param [String] file file name
+      #
+      # @return [void]
+      #
+      def create_instance_relation(bib, file)
+        inst = parse_file file
+        create_relation(inst, "hasInstance") { |rel| bib.relation << rel }
+        create_relation(bib, "instanceOf") { |rel| inst.relation << rel }
+        File.write file, serialize(inst), encoding: "UTF-8"
+      end
+
+      #
+      # Create relation
+      #
+      # @param [Relaton::Ccsds::Item] bib the related bibliographic item
+      # @param [String] type type of relation
+      #
+      # @return [Relaton::Bib::Relation] relation
+      #
+      def create_relation(bib, type)
+        bib_docid = bib.docidentifier.first
+        return unless bib_docid
+
+        docid = Bib::Docidentifier.from_yaml(bib_docid.to_yaml)
+        rel = Relaton::Bib::ItemData.new docidentifier: [docid], formattedref: Relaton::Bib::Formattedref.new(content: bib_docid.content.dup)
+        yield Relaton::Bib::Relation.new(type: type, bibitem: rel)
+      end
+
+      #
+      # Merge identical documents with different links (updaes given bibitem)
+      #
+      # @param [Relaton::Ccsds::Item] bib bibliographic item
+      # @param [String] file path to existing document
+      #
+      # @return [void]
+      #
+      def merge_links(bib, file) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        # skip merging when new file
+        unless @files.include?(file)
+          @files << file
+          return
+        end
+
+        puts "(#{file}) file already exists. Trying to merge links ..."
+
+        bib2 = parse_file file
+        bib2.source.each do |src|
+          next if bib.source.any? { |s| s.type == src.type }
+
+          bib.source << src
+        end
+        Util.info "links are merged.", key: file
+      end
+
+      def to_yaml(bib) = bib.to_yaml
+      def to_xml(bib) = bib.to_xml(bibdata: true)
+      def to_bibxml(bib) = bib.to_rfcxml
+    end
+  end
+end

data/lib/relaton/ccsds/data/iso_references.rb

@@ -0,0 +1,30 @@
+require "csv"
+require "singleton"
+require "mechanize"
+
+module Relaton
+  module Ccsds
+    class IsoReferences
+      include Singleton
+
+      ISO_CSV_URL = "https://isopublicstorageprod.blob.core.windows.net/opendata/_latest"\
+                    "/iso_deliverables_metadata/csv/iso_deliverables_metadata.csv".freeze
+
+      def [](key)
+        data[key]
+      end
+
+      private
+
+      def data
+        @data ||= begin
+          csv = Mechanize.new.get(ISO_CSV_URL).body
+          rows = CSV.parse(csv, headers: true)
+          rows.each_with_object({}) do |row, h|
+            h[row["id"]] = row["reference"]
+          end
+        end
+      end
+    end
+  end
+end