relaton-calconnect 1.9.0 → 1.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c3a78f7962d9b70c4dc6a30bd002f9973998f55d52af6612f63beb58e5109bf
-  data.tar.gz: e0f8ac1a2c0c676dca4e1f3ebf0ae044a453a143824f953e860017777b01a587
+  metadata.gz: 276dc8eec3c63bb0992e5986afcf1c45264e4541c5f008697c51f758be583d94
+  data.tar.gz: 50512e178a70c19233a71a7ecfd081efa23128227e4ec9de73d4d270a8c1a1aa
 SHA512:
-  metadata.gz: 66e83c6550539dcacb9583e9f90d67666b70ac9e39b8802a737741600b88ee3c1ca11c7cd9275acccc9ef023e9602a3495f573f75d76845070864f4335e64e8b
-  data.tar.gz: c7ace3d658e7c003847a1cc46b11385cf868d9fc47812e57c46d8ce08a87f865aa61a63b3e5724cc4a9c4554ca90d014cc7cdd7627ed9d09444c07aeba168c19
+  metadata.gz: 3786fc11fc5a004f691d3f5994bd4289ab7f71116cfb0de5f5e3d2477a6460a7b3b4a7073b91bdd9b736f046bad68e824fa22cb983727589a89899d168419615
+  data.tar.gz: a0032955308db628a798ee1f9a45616e49e82799031ee205de6b73298a768fee8ef175835330567ef5709c8a2ba5d853ca0d10c865fdc793baf966c472bb158c

data/README.adoc

@@ -106,6 +106,25 @@ RelatonCalconnect::CcBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+This gem uses the https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml dataset as one of data sources.
+
+The method `RelatonCalconnect::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the datast and save them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonCalconnect::DataFetcher.fetch
+Started at: 2021-09-09 16:03:51 +0200
+Stopped at: 2021-09-09 16:04:12 +0200
+Done in: 20 sec.
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/relaton_calconnect/data_fetcher.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal:true
+
+module RelatonCalconnect
+  #
+  # Relaton-calconnect data fetcher
+  #
+  class DataFetcher
+    # DOMAIN = "https://standards.calconnect.org/"
+    # SCHEME, HOST = DOMAIN.split(%r{:?/?/})
+    ENDPOINT = "https://standards.calconnect.org/relaton/index.yaml"
+    # DATADIR = "data"
+    # DATAFILE = File.join DATADIR, "bibliography.yml"
+    # ETAGFILE = File.join DATADIR, "etag.txt"
+
+    def initialize(output, format)
+      @output = output
+      @etagfile = File.join output, "etag.txt"
+      @format = format
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # fetch data form server and save it to file.
+    #
+    def fetch
+      resp = Faraday.new(ENDPOINT, headers: { "If-None-Match" => etag }).get
+      # return if there aren't any changes since last fetching
+      return unless resp.status == 200
+
+      data = YAML.safe_load resp.body
+      all_success = true
+      data["root"]["items"].each do |doc|
+        success = parse_page doc
+        all_success &&= success
+      end
+      self.etag = resp[:etag] if all_success
+    end
+
+    private
+
+    #
+    # Parse document and write it to file
+    #
+    # @param [Hash] doc
+    #
+    def parse_page(doc)
+      bib = Scrapper.parse_page doc
+      # bib.link.each { |l| l.content.merge!(scheme: SCHEME, host: HOST) unless l.content.host }
+      write_doc doc["docid"]["id"], bib
+      true
+    rescue StandardError => e
+      warn "Document: #{doc['docid']['id']}"
+      warn e.message
+      puts e.backtrace
+      false
+    end
+
+    def write_doc(docid, bib)
+      content = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
+      file = File.join @output, "#{docid.downcase.gsub(%r{[/\s:]}, '_')}.#{@format}"
+      # if File.exist? file
+      #   warn "#{file} exist"
+      # else
+      File.write file, content, encoding: "UTF-8"
+      # end
+    end
+
+    #
+    # Read ETag from file
+    #
+    # @return [String, NilClass]
+    def etag
+      @etag ||= File.exist?(@etagfile) ? File.read(@etagfile, encoding: "UTF-8") : nil
+    end
+
+    #
+    # Save ETag to file
+    #
+    # @param tag [String]
+    def etag=(e_tag)
+      File.write @etagfile, e_tag, encoding: "UTF-8"
+    end
+  end
+end

data/lib/relaton_calconnect/processor.rb

@@ -4,11 +4,12 @@ module RelatonCalconnect
   class Processor < Relaton::Processor
     attr_reader :idtype
 
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_calconnect
       @prefix = "CC"
       @defaultprefix = %r{^CC\s}
       @idtype = "CC"
+      @datasets = %w[calconnect-org]
     end
 
     # @param code [String]
@@ -19,6 +20,18 @@ module RelatonCalconnect
       ::RelatonCalconnect::CcBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonCalconnect::CcBibliographicItem]
     def from_xml(xml)

data/lib/relaton_calconnect/scrapper.rb

@@ -1,17 +1,24 @@
 module RelatonCalconnect
   module Scrapper
     DOMAIN = "https://standards.calconnect.org/".freeze
+    SCHEME, HOST = DOMAIN.split(%r{:?/?/})
     # DOMAIN = "http://127.0.0.1:4000/".freeze
 
     class << self
       # papam hit [Hash]
       # @return [RelatonOgc::OrcBibliographicItem]
       def parse_page(hit)
-        link = hit["link"].detect { |l| l["type"] == "rxl" }
+        links = array(hit["link"])
+        link = links.detect { |l| l["type"] == "rxl" }
         if link
-          bib_xml = fetch_bib_xml link["content"]
-          XMLParser.from_xml bib_xml
+          bib = fetch_bib_xml link["content"]
+          update_links bib, links
+          # XMLParser.from_xml bib_xml
+        else
+          bib = RelatonCalconnect::CcBibliographicItem.from_hash doc_to_hash(hit)
         end
+        bib.link.each { |l| l.content.merge!(scheme: SCHEME, host: HOST) unless l.content.host }
+        bib
       end
 
       private
@@ -19,15 +26,25 @@ module RelatonCalconnect
       # @param url [String]
       # @return [String] XML
       def fetch_bib_xml(url)
+        # rxl = get_rxl url
+        # uri_rxl = rxl.at("uri[@type='rxl']")
+        # return rxl.to_xml unless uri_rxl
+
+        # uri_xml = rxl.xpath("//uri").to_xml
+        # rxl = get_rxl uri_rxl.text
+        # docid = rxl.at "//docidentifier"
+        # docid.add_previous_sibling uri_xml
+        # rxl.to_xml
         rxl = get_rxl url
         uri_rxl = rxl.at("uri[@type='rxl']")
-        return rxl.to_xml unless uri_rxl
-
-        uri_xml = rxl.xpath("//uri").to_xml
-        rxl = get_rxl uri_rxl.text
-        docid = rxl.at "//docidentifier"
-        docid.add_previous_sibling uri_xml
-        rxl.to_xml
+        if uri_rxl
+          uri_xml = rxl.xpath("//uri").to_xml
+          rxl = get_rxl uri_rxl.text
+          docid = rxl.at "//docidentifier"
+          docid.add_previous_sibling uri_xml
+        end
+        xml = rxl.to_xml.gsub!(%r{(</?)technical-committee(>)}, '\1committee\2')
+        RelatonCalconnect::XMLParser.from_xml xml
       end
 
       # @param path [String]
@@ -36,6 +53,44 @@ module RelatonCalconnect
         resp = Faraday.get DOMAIN + path
         Nokogiri::XML resp.body
       end
+
+      #
+      # Fix editorial group
+      #
+      # @param [Hash] doc
+      #
+      # @return [Hash]
+      #
+      def doc_to_hash(doc)
+        array(doc["editorialgroup"]).each do |eg|
+          tc = eg.delete("technical_committee")
+          eg.merge!(tc) if tc
+        end
+        doc
+      end
+
+      def update_links(bib, links)
+        links.each do |l|
+          tu = l.transform_keys(&:to_sym)
+          bib.link << RelatonBib::TypedUri.new(**tu) unless bib.url(l["type"])
+        end
+        bib
+      end
+
+      #
+      # Wrap into Array if not Array
+      #
+      # @param [Array, Hash, String, nil] content
+      #
+      # @return [Array<Hash, String>]
+      #
+      def array(content)
+        case content
+        when Array then content
+        when nil then []
+        else [content]
+        end
+      end
     end
   end
 end

data/lib/relaton_calconnect/version.rb

@@ -1,3 +1,3 @@
 module RelatonCalconnect
-  VERSION = "1.9.0".freeze
+  VERSION = "1.9.1".freeze
 end

data/lib/relaton_calconnect.rb

@@ -8,6 +8,7 @@ require "relaton_calconnect/technical_committee"
 require "relaton_calconnect/cc_bibliographic_item"
 require "relaton_calconnect/xml_parser"
 require "relaton_calconnect/hash_converter"
+require "relaton_calconnect/data_fetcher"
 
 module RelatonCalconnect
   class Error < StandardError; end

data/relaton_calconnect.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "ruby-jing"
   spec.add_development_dependency "simplecov"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-calconnect
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.9.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-26 00:00:00.000000000 Z
+date: 2021-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -163,6 +163,7 @@ files:
 - lib/relaton_calconnect.rb
 - lib/relaton_calconnect/cc_bibliographic_item.rb
 - lib/relaton_calconnect/cc_bibliography.rb
+- lib/relaton_calconnect/data_fetcher.rb
 - lib/relaton_calconnect/hash_converter.rb
 - lib/relaton_calconnect/hit.rb
 - lib/relaton_calconnect/hit_collection.rb