relaton-calconnect 1.14.1 → 1.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e56768bb659378f291f2bd2ec6f57d46ff0db713c6465150b57e3a3ba85aeff0
-  data.tar.gz: db3c391bd354c06cbb774e2f13f0f746ec615bee53c139876f70e312816fce19
+  metadata.gz: 181a7e1a48eb95ff2558f0895621f59fb8a52f167c0ccac81433f5346e52e38a
+  data.tar.gz: a24dae24e0803340a07ddf3882a0343bb8ba52f05bb2d923df016d33dfd4e7cb
 SHA512:
-  metadata.gz: bb8b6b662b644931211db146edb0e4470d068f9c94987f7594793656655f450902b3e601217f9ecf24975cf7a6434150c5c67c668c18a0ecc6464653d07edbe1
-  data.tar.gz: a373c880ef6c73092333b85e7f686d765d596ef8bf6be2516124393d63fc179b64fa0005f69a747ccc139bde5117c5d8f2b0937a381cf561e2664894ee9b021a
+  metadata.gz: aab8cae2f19844f9a7d8d999b568d16f1a22344d88d3df81f924ef0e7c225f58d0c05616b6f6b73e3a6d3b8b116bc3df7c33e5ca3f3cd187391c1b7d4d4163fc
+  data.tar.gz: 12b98b7c9ae142bebd87bf4201889c471f28b5c477dc364936e0d94d3c6a3a0c1dd4a8a01cce9367b44183a2fcde70bdde43a50d2f58da04433318ab2f58939a

data/Gemfile

@@ -3,6 +3,9 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in relaton_calconnect.gemspec
 gemspec
 
+gem "equivalent-xml", "~> 0.6"
+gem "rake", "~> 13.0"
+gem "rspec", "~> 3.0"
 gem "ruby-jing"
 gem "simplecov"
 gem "vcr"

data/README.adoc

@@ -50,7 +50,7 @@ item.to_xml
       ...
     </bibitem>"
 ----
-With argument `bibdata: true` it ouputs XML wrapped by `bibdata` element and adds flavour `ext` element.
+With argument `bibdata: true` it outputs XML wrapped by `bibdata` element and adds flavour `ext` element.
 [source,ruby]
 ----
 item.to_xml bibdata: true
@@ -87,7 +87,7 @@ item.link
  #<RelatonBib::TypedUri:0x00007ff2d4b23a08 @content=#<Addressable::URI:0x6b8 URI:https://standards.calconnect.org/csd/cc-10005.rxl>, @type="rxl">]
 ----
 
-=== Get code, and year
+=== Get document by code and year
 [source,ruby]
 ----
 RelatonCalconnect::CcBibliography.get "CC/DIR 10005", "2019", {}
@@ -119,13 +119,13 @@ RelatonCalconnect::CcBibliographicItem.from_hash hash
 
 === Fetch data
 
-This gem uses the https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml dataset as one of data sources.
+This gem uses the https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml dataset as one of the data sources.
 
-The method `RelatonCalconnect::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the datast and save them to the `./data` folder in YAML format.
+The method `RelatonCalconnect::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the dataset and saves them to the `./data` folder in YAML format.
 Arguments:
 
 - `output` - folder to save documents (default './data').
-- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml` (default `yaml`).
+- `format` - the format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxml` (default `yaml`).
 
 [source,ruby]
 ----

data/lib/relaton_calconnect/cc_bibliography.rb

@@ -74,11 +74,11 @@ module RelatonCalconnect
       # @param missed_years [Array<Strig>]
       def fetch_ref_err(code, year, missed_years)
         id = year ? "#{code} year #{year}" : code
-        warn "[relaton-calconnect] WARNING: no match found online for #{id}. "\
-          "The code must be exactly like it is on the standards website."
+        warn "[relaton-calconnect] WARNING: no match found online for #{id}. " \
+             "The code must be exactly like it is on the standards website."
         unless missed_years.empty?
-          warn "[relaton-calconnect] (There was no match for #{year}, though "\
-            "there were matches found for #{missed_years.join(', ')}.)"
+          warn "[relaton-calconnect] (There was no match for #{year}, though " \
+               "there were matches found for #{missed_years.join(', ')}.)"
         end
         nil
       end

data/lib/relaton_calconnect/data_fetcher.rb

@@ -16,8 +16,19 @@ module RelatonCalconnect
       @output = output
       @etagfile = File.join output, "etag.txt"
       @format = format
+      @ext = format.sub "bibxml", "xml"
+      @files = []
+      @index = Relaton::Index.find_or_create :CC, file: "index-v1.yaml"
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] output directory to output documents, default: "data"
+    # @param [String] format output format, default: "yaml"
+    #
+    # @return [void]
+    #
     def self.fetch(output: "data", format: "yaml")
       t1 = Time.now
       puts "Started at: #{t1}"
@@ -31,18 +42,16 @@ module RelatonCalconnect
     #
     # fetch data form server and save it to file.
     #
-    def fetch
+    def fetch # rubocop:disable Metrics/AbcSize
       resp = Faraday.new(ENDPOINT, headers: { "If-None-Match" => etag }).get
       # return if there aren't any changes since last fetching
       return unless resp.status == 200
 
       data = YAML.safe_load resp.body
       all_success = true
-      data["root"]["items"].each do |doc|
-        success = parse_page doc
-        all_success &&= success
-      end
+      data["root"]["items"].each { |doc| all_success &&= parse_page doc }
       self.etag = resp[:etag] if all_success
+      @index.save
     end
 
     private
@@ -64,14 +73,20 @@ module RelatonCalconnect
       false
     end
 
-    def write_doc(docid, bib)
-      content = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
-      file = File.join @output, "#{docid.upcase.gsub(%r{[/\s:]}, '_')}.#{@format}"
-      # if File.exist? file
-      #   warn "#{file} exist"
-      # else
+    def write_doc(docid, bib) # rubocop:disable Metrics/MethodLength
+      content = case @format
+                when "xml" then bib.to_xml(bibdata: true)
+                when "bibxml" then bib.to_bibxml
+                else bib.to_hash.to_yaml
+                end
+      file = File.join @output, "#{docid.upcase.gsub(%r{[/\s:]}, '_')}.#{@ext}"
+      if @files.include? file
+        warn "#{file} exist"
+      else
+        @files << file
+      end
+      @index.add_or_update docid, file
       File.write file, content, encoding: "UTF-8"
-      # end
     end
 
     #

data/lib/relaton_calconnect/hit.rb

@@ -3,7 +3,12 @@ module RelatonCalconnect
     # Parse page.
     # @return [RelatonCalconnect::CcBliographicItem]
     def fetch
-      @fetch ||= Scrapper.parse_page @hit
+      # @fetch ||= Scrapper.parse_page @hit
+      @fetch ||= begin
+        url = "#{HitCollection::GHURL}#{@hit[:file]}"
+        resp = Faraday.get url
+        CcBibliographicItem.from_hash YAML.safe_load(resp.body)
+      end
     end
   end
 end

data/lib/relaton_calconnect/hit_collection.rb

@@ -4,23 +4,29 @@ require "fileutils"
 
 module RelatonCalconnect
   class HitCollection < RelatonBib::HitCollection
-    ENDPOINT = "https://standards.calconnect.org/relaton/index.yaml".freeze
+    # ENDPOINT = "https://standards.calconnect.org/relaton/index.yaml".freeze
     # ENDPOINT = "http://127.0.0.1:4000/relaton/index.yaml".freeze
-    DATADIR = File.expand_path ".relaton/calconnect", Dir.home
-    DATAFILE = File.expand_path "bibliography.yml", DATADIR
-    ETAGFILE = File.expand_path "etag.txt", DATADIR
+    # DATADIR = File.expand_path ".relaton/calconnect", Dir.home
+    # DATAFILE = File.expand_path "bibliography.yml", DATADIR
+    # ETAGFILE = File.expand_path "etag.txt", DATADIR
+    GHURL = "https://raw.githubusercontent.com/relaton/relaton-data-calconnect/main/".freeze
+    INDEX_FILE = "index-v1.yaml".freeze
 
     # @param ref [Strig]
     # @param year [String]
     # @param opts [Hash]
     def initialize(ref, year = nil)
       super
-      @array = from_yaml(ref).sort_by do |hit|
-        hit.hit["revdate"] ? Date.parse(hit.hit["revdate"]) : Date.new
-      end.reverse
+      # @array = from_yaml(ref).sort_by do |hit|
+      #   hit.hit["revdate"] ? Date.parse(hit.hit["revdate"]) : Date.new
+      # end.reverse
+      index = Relaton::Index.find_or_create :CC, url: "#{GHURL}index-v1.zip", file: INDEX_FILE
+      @array = index.search(ref).map do |row|
+        Hit.new(row, self)
+      end
     end
 
-    private
+    # private
 
     #
     # Fetch data from yaml
@@ -29,52 +35,52 @@ module RelatonCalconnect
     #
     # @return [Array<RelatonBib::Hit>]
     #
-    def from_yaml(docid, **_opts)
-      data["root"]["items"].select do |doc|
-        doc["docid"] && doc["docid"]["id"].include?(docid)
-      end.map { |h| Hit.new(h, self) }
-    end
+    # def from_yaml(docid, **_opts)
+    #   data["root"]["items"].select do |doc|
+    #     doc["docid"] && doc["docid"]["id"].include?(docid)
+    #   end.map { |h| Hit.new(h, self) }
+    # end
 
     #
     # Fetches YAML data
     #
     # @return [Hash]
-    def data
-      FileUtils.mkdir_p DATADIR
-      ctime = File.ctime DATAFILE if File.exist? DATAFILE
-      fetch_data if !ctime || ctime.to_date < Date.today
-      @data ||= YAML.safe_load File.read(DATAFILE, encoding: "UTF-8")
-    end
+    # def data
+    #   FileUtils.mkdir_p DATADIR
+    #   ctime = File.ctime DATAFILE if File.exist? DATAFILE
+    #   fetch_data if !ctime || ctime.to_date < Date.today
+    #   @data ||= YAML.safe_load File.read(DATAFILE, encoding: "UTF-8")
+    # end
 
     #
     # fetch data from server and save it to file.
     #
-    def fetch_data
-      resp = Faraday.new(ENDPOINT, headers: { "If-None-Match" => etag }).get
-      # return if there aren't any changes since last fetching
-      return unless resp.status == 200
+    # def fetch_data
+    #   resp = Faraday.new(ENDPOINT, headers: { "If-None-Match" => etag }).get
+    #   # return if there aren't any changes since last fetching
+    #   return unless resp.status == 200
 
-      self.etag = resp[:etag]
-      @data = YAML.safe_load resp.body
-      File.write DATAFILE, @data.to_yaml, encoding: "UTF-8"
-    end
+    #   self.etag = resp[:etag]
+    #   @data = YAML.safe_load resp.body
+    #   File.write DATAFILE, @data.to_yaml, encoding: "UTF-8"
+    # end
 
     #
     # Read ETag from file
     #
     # @return [String, NilClass]
-    def etag
-      @etag ||= if File.exist? ETAGFILE
-                  File.read ETAGFILE, encoding: "UTF-8"
-                end
-    end
+    # def etag
+    #   @etag ||= if File.exist? ETAGFILE
+    #               File.read ETAGFILE, encoding: "UTF-8"
+    #             end
+    # end
 
     #
     # Save ETag to file
     #
     # @param tag [String]
-    def etag=(e_tag)
-      File.write ETAGFILE, e_tag, encoding: "UTF-8"
-    end
+    # def etag=(e_tag)
+    #   File.write ETAGFILE, e_tag, encoding: "UTF-8"
+    # end
   end
 end

data/lib/relaton_calconnect/processor.rb

@@ -49,5 +49,12 @@ module RelatonCalconnect
     def grammar_hash
       @grammar_hash ||= ::RelatonCalconnect.grammar_hash
     end
+
+    #
+    # Remove index file
+    #
+    def remove_index_file
+      Relaton::Index.find_or_create(:CC, url: true, file: HitCollection::INDEX_FILE).remove_file
+    end
   end
 end

data/lib/relaton_calconnect/scrapper.rb

@@ -47,7 +47,7 @@ module RelatonCalconnect
           docid = rxl.at "//docidentifier"
           docid.add_previous_sibling uri_xml
         end
-        xml = rxl.to_xml.gsub!(%r{(</?)technical-committee(>)}, '\1committee\2')
+        xml = rxl.to_xml.gsub(%r{(</?)technical-committee(>)}, '\1committee\2')
           .gsub(%r{type="(?:csd|CC)"(?=>)}i, '\0 primary="true"')
         RelatonCalconnect::XMLParser.from_xml xml
       end

data/lib/relaton_calconnect/version.rb

@@ -1,3 +1,3 @@
 module RelatonCalconnect
-  VERSION = "1.14.1".freeze
+  VERSION = "1.14.2".freeze
 end

data/lib/relaton_calconnect.rb

@@ -1,3 +1,4 @@
+require "relaton/index"
 require "relaton_bib"
 require "relaton_calconnect/version"
 require "relaton_calconnect/cc_bibliography"

data/relaton_calconnect.gemspec

@@ -26,10 +26,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
-  spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-
   spec.add_dependency "faraday", "~> 2.7.0"
   spec.add_dependency "relaton-bib", "~> 1.14.0"
+  spec.add_dependency "relaton-index", "~> 0.2.0"
 end

metadata

@@ -1,85 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: relaton-calconnect
 version: !ruby/object:Gem::Version
-  version: 1.14.1
+  version: 1.14.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-05-20 00:00:00.000000000 Z
+date: 2023-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: equivalent-xml
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
+  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
+        version: 2.7.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 2.7.0
 - !ruby/object:Gem::Dependency
-  name: faraday
+  name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.7.0
+        version: 1.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.7.0
+        version: 1.14.0
 - !ruby/object:Gem::Dependency
-  name: relaton-bib
+  name: relaton-index
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: 0.2.0
 description: 'RelatonIso: retrieve CC Standards for bibliographic use using the IsoBibliographicItem
   model'
 email: