relaton-calconnect 1.7.0 → 1.9.1

data/grammars/reqt.rng

@@ -30,15 +30,34 @@
         <data type="boolean"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="number"/>
+    </optional>
     <optional>
       <attribute name="subsequence"/>
     </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
     <attribute name="id">
       <data type="ID"/>
     </attribute>
     <optional>
       <attribute name="filename"/>
     </optional>
+    <optional>
+      <attribute name="model"/>
+    </optional>
+    <optional>
+      <attribute name="type"/>
+    </optional>
     <optional>
       <ref name="reqtitle"/>
     </optional>
@@ -48,9 +67,9 @@
     <optional>
       <ref name="subject"/>
     </optional>
-    <optional>
+    <zeroOrMore>
       <ref name="reqinherit"/>
-    </optional>
+    </zeroOrMore>
     <zeroOrMore>
       <ref name="classification"/>
     </zeroOrMore>
@@ -135,6 +154,16 @@
         <data type="boolean"/>
       </attribute>
     </optional>
+    <optional>
+      <attribute name="keep-with-next">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
+    <optional>
+      <attribute name="keep-lines-together">
+        <data type="boolean"/>
+      </attribute>
+    </optional>
     <oneOrMore>
       <ref name="BasicBlock"/>
     </oneOrMore>

data/lib/relaton_calconnect/cc_bibliographic_item.rb

@@ -4,5 +4,12 @@ module RelatonCalconnect
       directive guide specification standard report administrative amendment
       technical\ corrigendum advisory
     ].freeze
+
+    # @param hash [Hash]
+    # @return [RelatonIsoBib::CcBibliographicItem]
+    def self.from_hash(hash)
+      item_hash = ::RelatonCalconnect::HashConverter.hash_to_bib(hash)
+      new(**item_hash)
+    end
   end
 end

data/lib/relaton_calconnect/cc_bibliography.rb

@@ -3,7 +3,7 @@ module RelatonCalconnect
     class << self
       # @param text [String]
       # @return [RelatonCalconnect::HitCollection]
-      def search(text, year = nil, opts = {})
+      def search(text, year = nil, _opts = {})
         HitCollection.new text, year
       rescue Faraday::ConnectionFailed
         raise RelatonBib::RequestError, "Could not access https://standards.calconnect.org"
@@ -18,11 +18,11 @@ module RelatonCalconnect
       # @option opts [TrueClass, FalseClass] :bibdata
       #
       # @return [RelatonCalconnect::CcBibliographicItem]
-      def get(ref, year = nil, opts = {})
+      def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
         code = ref
 
         if year.nil?
-          /^(?<code1>[^\s]+(\s\w+)?\s[\d-]+):?(?<year1>\d{4})?/ =~ ref
+          /^(?<code1>[^\s]+(?:\s\w+)?\s[\d-]+):?(?<year1>\d{4})?/ =~ ref
           unless code1.nil?
             code = code1
             year = year1
@@ -30,7 +30,7 @@ module RelatonCalconnect
         end
 
         warn "[relaton-calconnect] (\"#{ref}\") fetching..."
-        result = bib_search_filter(code, year, opts) || (return nil)
+        result = search(code, year, opts) || (return nil)
         ret = bib_results_filter(result, year)
         if ret[:ret]
           warn "[relaton-calconnect] (\"#{ref}\") found #{ret[:ret].docidentifier.first.id}"
@@ -42,10 +42,6 @@ module RelatonCalconnect
 
       private
 
-      def bib_search_filter(code, year, opts)
-        search(code, year, opts)
-      end
-
       # Sort through the results from RelatonNist, fetching them three at a time,
       # and return the first result that matches the code,
       # matches the year (if provided), and which # has a title (amendments do not).
@@ -80,8 +76,8 @@ module RelatonCalconnect
         warn "[relaton-calconnect] WARNING: no match found online for #{id}. "\
           "The code must be exactly like it is on the standards website."
         unless missed_years.empty?
-          warn "[relaton-calconnect] (There was no match for #{year}, though there were matches "\
-            "found for #{missed_years.join(', ')}.)"
+          warn "[relaton-calconnect] (There was no match for #{year}, though "\
+            "there were matches found for #{missed_years.join(', ')}.)"
         end
         nil
       end

data/lib/relaton_calconnect/data_fetcher.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal:true
+
+module RelatonCalconnect
+  #
+  # Relaton-calconnect data fetcher
+  #
+  class DataFetcher
+    # DOMAIN = "https://standards.calconnect.org/"
+    # SCHEME, HOST = DOMAIN.split(%r{:?/?/})
+    ENDPOINT = "https://standards.calconnect.org/relaton/index.yaml"
+    # DATADIR = "data"
+    # DATAFILE = File.join DATADIR, "bibliography.yml"
+    # ETAGFILE = File.join DATADIR, "etag.txt"
+
+    def initialize(output, format)
+      @output = output
+      @etagfile = File.join output, "etag.txt"
+      @format = format
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      t1 = Time.now
+      puts "Started at: #{t1}"
+      FileUtils.mkdir_p output unless Dir.exist? output
+      new(output, format).fetch
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    end
+
+    #
+    # fetch data form server and save it to file.
+    #
+    def fetch
+      resp = Faraday.new(ENDPOINT, headers: { "If-None-Match" => etag }).get
+      # return if there aren't any changes since last fetching
+      return unless resp.status == 200
+
+      data = YAML.safe_load resp.body
+      all_success = true
+      data["root"]["items"].each do |doc|
+        success = parse_page doc
+        all_success &&= success
+      end
+      self.etag = resp[:etag] if all_success
+    end
+
+    private
+
+    #
+    # Parse document and write it to file
+    #
+    # @param [Hash] doc
+    #
+    def parse_page(doc)
+      bib = Scrapper.parse_page doc
+      # bib.link.each { |l| l.content.merge!(scheme: SCHEME, host: HOST) unless l.content.host }
+      write_doc doc["docid"]["id"], bib
+      true
+    rescue StandardError => e
+      warn "Document: #{doc['docid']['id']}"
+      warn e.message
+      puts e.backtrace
+      false
+    end
+
+    def write_doc(docid, bib)
+      content = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
+      file = File.join @output, "#{docid.downcase.gsub(%r{[/\s:]}, '_')}.#{@format}"
+      # if File.exist? file
+      #   warn "#{file} exist"
+      # else
+      File.write file, content, encoding: "UTF-8"
+      # end
+    end
+
+    #
+    # Read ETag from file
+    #
+    # @return [String, NilClass]
+    def etag
+      @etag ||= File.exist?(@etagfile) ? File.read(@etagfile, encoding: "UTF-8") : nil
+    end
+
+    #
+    # Save ETag to file
+    #
+    # @param tag [String]
+    def etag=(e_tag)
+      File.write @etagfile, e_tag, encoding: "UTF-8"
+    end
+  end
+end

data/lib/relaton_calconnect/hash_converter.rb

@@ -6,7 +6,7 @@ module RelatonCalconnect
         return unless ret[:editorialgroup]
 
         technical_committee = array(ret[:editorialgroup]).map do |wg|
-          TechnicalCommittee.new RelatonBib::WorkGroup.new(wg)
+          TechnicalCommittee.new RelatonBib::WorkGroup.new(**wg)
         end
         ret[:editorialgroup] = RelatonBib::EditorialGroup.new technical_committee
       end

data/lib/relaton_calconnect/hit_collection.rb

@@ -23,7 +23,7 @@ module RelatonCalconnect
     private
 
     #
-    # Fetch data form yaml
+    # Fetch data from yaml
     #
     # @param docid [String]
     def from_yaml(docid, **_opts)
@@ -44,7 +44,7 @@ module RelatonCalconnect
     end
 
     #
-    # fetch data form server and save it to file.
+    # fetch data from server and save it to file.
     #
     def fetch_data
       resp = Faraday.new(ENDPOINT, headers: { "If-None-Match" => etag }).get
@@ -57,7 +57,7 @@ module RelatonCalconnect
     end
 
     #
-    # Read ETag form file
+    # Read ETag from file
     #
     # @return [String, NilClass]
     def etag

data/lib/relaton_calconnect/processor.rb

@@ -4,11 +4,12 @@ module RelatonCalconnect
   class Processor < Relaton::Processor
     attr_reader :idtype
 
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_calconnect
       @prefix = "CC"
       @defaultprefix = %r{^CC\s}
       @idtype = "CC"
+      @datasets = %w[calconnect-org]
     end
 
     # @param code [String]
@@ -19,6 +20,18 @@ module RelatonCalconnect
       ::RelatonCalconnect::CcBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonCalconnect::CcBibliographicItem]
     def from_xml(xml)
@@ -28,8 +41,7 @@ module RelatonCalconnect
     # @param hash [Hash]
     # @return [RelatonIsoBib::CcBibliographicItem]
     def hash_to_bib(hash)
-      item_hash = ::RelatonCalconnect::HashConverter.hash_to_bib(hash)
-      ::RelatonCalconnect::CcBibliographicItem.new item_hash
+      ::RelatonCalconnect::CcBibliographicItem.from_hash hash
     end
 
     # Returns hash of XML grammar

data/lib/relaton_calconnect/scrapper.rb

@@ -1,17 +1,24 @@
 module RelatonCalconnect
   module Scrapper
     DOMAIN = "https://standards.calconnect.org/".freeze
+    SCHEME, HOST = DOMAIN.split(%r{:?/?/})
     # DOMAIN = "http://127.0.0.1:4000/".freeze
 
     class << self
       # papam hit [Hash]
       # @return [RelatonOgc::OrcBibliographicItem]
       def parse_page(hit)
-        link = hit["link"].detect { |l| l["type"] == "rxl" }
+        links = array(hit["link"])
+        link = links.detect { |l| l["type"] == "rxl" }
         if link
-          bib_xml = fetch_bib_xml link["content"]
-          XMLParser.from_xml bib_xml
+          bib = fetch_bib_xml link["content"]
+          update_links bib, links
+          # XMLParser.from_xml bib_xml
+        else
+          bib = RelatonCalconnect::CcBibliographicItem.from_hash doc_to_hash(hit)
         end
+        bib.link.each { |l| l.content.merge!(scheme: SCHEME, host: HOST) unless l.content.host }
+        bib
       end
 
       private
@@ -19,15 +26,25 @@ module RelatonCalconnect
       # @param url [String]
       # @return [String] XML
       def fetch_bib_xml(url)
+        # rxl = get_rxl url
+        # uri_rxl = rxl.at("uri[@type='rxl']")
+        # return rxl.to_xml unless uri_rxl
+
+        # uri_xml = rxl.xpath("//uri").to_xml
+        # rxl = get_rxl uri_rxl.text
+        # docid = rxl.at "//docidentifier"
+        # docid.add_previous_sibling uri_xml
+        # rxl.to_xml
         rxl = get_rxl url
         uri_rxl = rxl.at("uri[@type='rxl']")
-        return rxl.to_xml unless uri_rxl
-
-        uri_xml = rxl.xpath("//uri").to_xml
-        rxl = get_rxl uri_rxl.text
-        docid = rxl.at "//docidentifier"
-        docid.add_previous_sibling uri_xml
-        rxl.to_xml
+        if uri_rxl
+          uri_xml = rxl.xpath("//uri").to_xml
+          rxl = get_rxl uri_rxl.text
+          docid = rxl.at "//docidentifier"
+          docid.add_previous_sibling uri_xml
+        end
+        xml = rxl.to_xml.gsub!(%r{(</?)technical-committee(>)}, '\1committee\2')
+        RelatonCalconnect::XMLParser.from_xml xml
       end
 
       # @param path [String]
@@ -36,6 +53,44 @@ module RelatonCalconnect
         resp = Faraday.get DOMAIN + path
         Nokogiri::XML resp.body
       end
+
+      #
+      # Fix editorial group
+      #
+      # @param [Hash] doc
+      #
+      # @return [Hash]
+      #
+      def doc_to_hash(doc)
+        array(doc["editorialgroup"]).each do |eg|
+          tc = eg.delete("technical_committee")
+          eg.merge!(tc) if tc
+        end
+        doc
+      end
+
+      def update_links(bib, links)
+        links.each do |l|
+          tu = l.transform_keys(&:to_sym)
+          bib.link << RelatonBib::TypedUri.new(**tu) unless bib.url(l["type"])
+        end
+        bib
+      end
+
+      #
+      # Wrap into Array if not Array
+      #
+      # @param [Array, Hash, String, nil] content
+      #
+      # @return [Array<Hash, String>]
+      #
+      def array(content)
+        case content
+        when Array then content
+        when nil then []
+        else [content]
+        end
+      end
     end
   end
 end

data/lib/relaton_calconnect/version.rb

@@ -1,3 +1,3 @@
 module RelatonCalconnect
-  VERSION = "1.7.0".freeze
+  VERSION = "1.9.1".freeze
 end

data/lib/relaton_calconnect/xml_parser.rb

@@ -5,7 +5,7 @@ module RelatonCalconnect
       # @param item_hash [Hash]
       # @return [RelatonIsoBib::IsoBibliographicItem]
       def bib_item(item_hash)
-        CcBibliographicItem.new item_hash
+        CcBibliographicItem.new(**item_hash)
       end
 
       # @param ext [Nokogiri::XML::Element]
@@ -14,7 +14,7 @@ module RelatonCalconnect
         return unless ext && (eg = ext.at "editorialgroup")
 
         eg = eg.xpath("committee", "technical-committee").map do |tc|
-          wg = RelatonBib::WorkGroup.new(content: tc.text, number: tc[:number]&.to_i,
+          wg = RelatonBib::WorkGroup.new(name: tc.text, number: tc[:number]&.to_i,
                                          type: tc[:type])
           TechnicalCommittee.new wg
         end

data/lib/relaton_calconnect.rb

@@ -8,6 +8,7 @@ require "relaton_calconnect/technical_committee"
 require "relaton_calconnect/cc_bibliographic_item"
 require "relaton_calconnect/xml_parser"
 require "relaton_calconnect/hash_converter"
+require "relaton_calconnect/data_fetcher"
 
 module RelatonCalconnect
   class Error < StandardError; end

data/relaton_calconnect.gemspec

@@ -24,18 +24,16 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
-  spec.add_development_dependency "debase"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "ruby-debug-ide"
   spec.add_development_dependency "ruby-jing"
   spec.add_development_dependency "simplecov"
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 
   spec.add_dependency "faraday"
-  spec.add_dependency "relaton-bib", "~> 1.7.0"
+  spec.add_dependency "relaton-bib", "~> 1.9.0"
 end