relaton-ogc 1.14.2 → 1.14.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73f9412fdea955cd429e42939749b8073fb4fed8c38e5f78ab0b5977de89afa0
-  data.tar.gz: ce2cacba7411ba37786e5e414538fd3add66758203d22c14c98c4baa9f5bb5aa
+  metadata.gz: df9aa5c8876eed257c8bba8f85d6f59e917f5239c17a8937f233209c2318a6cc
+  data.tar.gz: 829cb72c24ba23c1762fb90f51f683fb889eaa8cb6a2b318de02932b5c4b07cc
 SHA512:
-  metadata.gz: deff577fbd2b83fcd787dd132d70bd35cec9f550aea0020bdcfccd3c1d3185a9a083610b9dc5c45f97026730bcb0fdfdcc8f5a581baa028daae025e1f4fb2a77
-  data.tar.gz: e8884898e4604fdaeaebcca4039c6a8662c7b503d4e2737e386bc47f2dc7f85ef05b96e7f5b105d6b3c0c0de63ca8219a918611a1d092aed8bd56db2bf26d7cf
+  metadata.gz: 8fcb76f8709b72a4e50f9a163d9e7234c08dd67690cf38b72395047f23347be69feff9c460392b95554889a9fdff40b1b737de892bb489f4632931830792d6c1
+  data.tar.gz: 6b4ebf6cb8538a543db51ca9dca4dba2fab47287229a2ce864516d063c6f634e653a6b25f8fe5817bf74301c4657d66efb1fd7631e56ea8ef5f48d45803e72b4

data/.github/workflows/rake.yml

@@ -5,6 +5,7 @@ name: rake
 on:
   push:
     branches: [ master, main ]
+    tags: [ v* ]
   pull_request:
 
 jobs:

data/.github/workflows/release.yml

@@ -10,8 +10,9 @@ on:
           Next release version. Possible values: x.y.z, major, minor, patch or pre|rc|etc
         required: true
         default: 'skip'
-  push:
-    tags: [ v* ]
+  repository_dispatch:
+    types: [ do-release ]
+
 
 jobs:
   release:

data/Gemfile

@@ -3,7 +3,10 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in relaton_ogc.gemspec
 gemspec
 
+gem "equivalent-xml", "~> 0.6"
 gem "pry-byebug"
+gem "rake", "~> 13.0"
+gem "rspec", "~> 3.0"
 gem "ruby-jing"
 gem "simplecov"
 gem "vcr"

data/lib/relaton_ogc/data_fetcher.rb

@@ -1,9 +1,9 @@
+# frozen_string_literal: true
+
 module RelatonOgc
   class DataFetcher
     module Utils
-      ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
-                 "NamingAuthority/master/incubation/bibliography/"\
-                 "bibliography.json".freeze
+      ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/NamingAuthority/master/definitions/docs/docs.json"
 
       def get_data # rubocop:disable Metrics/AbcSize
         h = {}
@@ -43,45 +43,54 @@ module RelatonOgc
     # Create DataFetcher instance
     #
     # @param [String] output directory to save the documents
-    # @param [String] format output format "yaml" or "xmo"
+    # @param [String] format output format "yaml", "xml", or "bibxml"
     #
     def initialize(output, format)
       @output = output
       @etagfile = File.join output, "etag.txt"
       @format = format
+      @ext = format.sub "bib", ""
       @docids = []
-      @dupids = []
+      @dupids = Set.new
+    end
+
+    def index
+      @index ||= Relaton::Index.find_or_create :ogc, file: "index-v1.yaml"
     end
 
     def self.fetch(output: "data", format: "yaml")
       t1 = Time.now
       puts "Started at: #{t1}"
-      FileUtils.mkdir_p output unless Dir.exist? output
+      FileUtils.mkdir_p output
       new(output, format).fetch
       t2 = Time.now
       puts "Stopped at: #{t2}"
       puts "Done in: #{(t2 - t1).round} sec."
     end
 
-    def fetch # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def fetch
       get_data do |etag, json|
         no_errors = true
-        json.each do |_, hit|
-          next if hit["type"] == "CC"
-
-          bib = Scrapper.parse_page hit
-          write_document bib
-        rescue StandardError => e
-          no_errors = false
-          warn "Fetching document: #{hit['identifier']}"
-          warn "#{e.class} #{e.message}"
-          warn e.backtrace
-        end
-        warn "[relaton-ogc] WARNING Duplicated documents: #{@dupids.uniq.join(', ')}" if @dupids.any?
+        json.each { |_, hit| fetch_doc(hit) || no_errors = false }
+        warn "[relaton-ogc] WARNING Duplicated documents: #{@dupids.to_a.join(', ')}" if @dupids.any?
         self.etag = etag if no_errors
+        index.save
       end
     end
 
+    def fetch_doc(hit)
+      return if hit["type"] == "CC"
+
+      bib = Scrapper.parse_page hit
+      write_document bib
+      true
+    rescue StandardError => e
+      warn "Fetching document: #{hit['identifier']}"
+      warn "#{e.class} #{e.message}"
+      warn e.backtrace
+      false
+    end
+
     def write_document(bib) # rubocop:disable Metrics/AbcSize
       if @docids.include?(bib.docidentifier[0].id)
         @dupids << bib.docidentifier[0].id
@@ -89,10 +98,22 @@ module RelatonOgc
       end
 
       @docids << bib.docidentifier[0].id
+      file = file_name bib
+      index.add_or_update bib.docidentifier[0].id, file
+      File.write file, content(bib), encoding: "UTF-8"
+    end
+
+    def file_name(bib)
       name = bib.docidentifier[0].id.upcase.gsub(/[\s:.]/, "_")
-      file = "#{@output}/#{name}.#{@format}"
-      content = @format == "xml" ? bib.to_xml(bibdata: true) : bib.to_hash.to_yaml
-      File.write file, content, encoding: "UTF-8"
+      "#{@output}/#{name}.#{@ext}"
+    end
+
+    def content(bib)
+      case @format
+      when "xml" then bib.to_xml bibdata: true
+      when "yaml" then bib.to_hash.to_yaml
+      when "bibxml" then bib.to_bibxml
+      end
     end
   end
 end

data/lib/relaton_ogc/hit_collection.rb

@@ -4,76 +4,29 @@ require "fileutils"
 
 module RelatonOgc
   class HitCollection < RelatonBib::HitCollection
-    # include DataFetcher::Utils
-
-    # ENDPOINT = "https://raw.githubusercontent.com/opengeospatial/"\
-    #            "NamingAuthority/master/incubation/bibliography/"\
-    #            "bibliography.json".freeze
-    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-ogc/main/data/".freeze
-    # DATADIR = File.expand_path ".relaton/ogc/", Dir.home
-    # DATAFILE = File.expand_path "bibliography.json", DATADIR
-    # ETAGFILE = File.expand_path "etag.txt", DATADIR
+    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-ogc/main/".freeze
+    INDEX_FILE = "index-v1.yaml".freeze
 
     # @param code [Strig]
     # @param year [String]
     # @param opts [Hash]
     def initialize(code, year = nil) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       super
-      # @etagfile = File.expand_path "etag.txt", DATADIR
-      # @array = from_json(ref).sort_by do |hit|
-      #   hit.hit["date"] ? Date.parse(hit.hit["date"]) : Date.new
-      # rescue ArgumentError
-      #   Date.parse "0000-01-01"
-      # end.reverse
-      url = "#{ENDPOINT}#{code.upcase.gsub(/[\s:.]/, '_')}.yaml"
-      resp = Faraday.get url do |req|
-        req.options.timeout = 10
-      end
-      @array = case resp.status
-               when 200
-                 hash = YAML.safe_load(resp.body)
-                 hash["fetched"] = Date.today.to_s
-                 bib = OgcBibliographicItem.from_hash hash
-                 [Hit.new(bib, self)]
-               else []
-               end
-    end
-
-    # private
+      @array = []
+      return if code.nil? || code.empty?
 
-    #
-    # Fetch data form json
-    #
-    # @param docid [String]
-    # def from_json(docid, **_opts)
-    #   ref = docid.sub(/^OGC\s/, "").strip
-    #   return [] if ref.empty?
+      index = Relaton::Index.find_or_create :ogc, url: "#{ENDPOINT}index-v1.zip", file: INDEX_FILE
+      row = index.search(code).min_by { |r| r[:id] }
+      return unless row
 
-    #   data.select do |_k, doc|
-    #     doc["type"] != "CC" && doc["identifier"].include?(ref)
-    #   end.map { |_k, h| Hit.new(h, self) }
-    # end
+      url = "#{ENDPOINT}#{row[:file]}"
+      resp = Faraday.get(url) { |req| req.options.timeout = 10 }
+      return unless resp.status == 200
 
-    #
-    # Fetches json data
-    #
-    # @return [Hash]
-    # def data
-    #   ctime = File.ctime DATAFILE if File.exist? DATAFILE
-    #   fetch_data if !ctime || ctime.to_date < Date.today
-    #   @data ||= JSON.parse File.read(DATAFILE, encoding: "UTF-8")
-    # end
-
-    #
-    # fetch data form server and save it to file.
-    #
-    # def fetch_data
-    #   json = get_data
-    #   return unless json
-
-    #   FileUtils.mkdir_p DATADIR unless Dir.exist? DATADIR
-    #   @data = json
-    #   File.write DATAFILE, @data.to_json, encoding: "UTF-8"
-    # end
+      hash = YAML.safe_load(resp.body)
+      hash["fetched"] = Date.today.to_s
+      bib = OgcBibliographicItem.from_hash hash
+      @array = [Hit.new(bib, self)]
+    end
   end
 end

data/lib/relaton_ogc/processor.rb

@@ -47,5 +47,12 @@ module RelatonOgc
     def grammar_hash
       @grammar_hash ||= ::RelatonOgc.grammar_hash
     end
+
+    #
+    # Remove index file
+    #
+    def remove_index_file
+      Relaton::Index.find_or_create(:ogc, url: true, file: HitCollection::INDEX_FILE).remove_file
+    end
   end
 end

data/lib/relaton_ogc/scrapper.rb

@@ -85,7 +85,7 @@ module RelatonOgc
       end
 
       # @param stage [String]
-      # @return [RelatonBib::DocumentStatus, NilClass]
+      # @return [RelatonBib::DocumentStatus, nil]
       def fetch_status(stage)
         stage && RelatonBib::DocumentStatus.new(stage: stage)
       end
@@ -134,12 +134,14 @@ module RelatonOgc
         )
       end
 
-      # @param date [String]
+      # @param date [String, nil]
       # @return [Array<RelatonBib::BibliographicDate>]
       def fetch_date(date)
         return [] unless date
 
         [RelatonBib::BibliographicDate.new(type: "published", on: date)]
+      rescue Date::Error
+        []
       end
     end
   end

data/lib/relaton_ogc/version.rb

@@ -1,3 +1,3 @@
 module RelatonOgc
-  VERSION = "1.14.2".freeze
+  VERSION = "1.14.3".freeze
 end

data/lib/relaton_ogc.rb

@@ -1,3 +1,4 @@
+require "relaton/index"
 require "relaton_iso_bib"
 require "relaton_ogc/version"
 require "relaton_ogc/ogc_bibliographic_item"

data/relaton_ogc.gemspec

@@ -25,10 +25,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
-  spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-
   spec.add_dependency "faraday", "~> 2.7.0"
+  spec.add_dependency "relaton-index", "~> 0.2.0"
   spec.add_dependency "relaton-iso-bib", "~> 1.14.0"
 end

metadata

@@ -1,71 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: relaton-ogc
 version: !ruby/object:Gem::Version
-  version: 1.14.2
+  version: 1.14.3
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-05-20 00:00:00.000000000 Z
+date: 2023-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: equivalent-xml
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
+  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
+        version: 2.7.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 2.7.0
 - !ruby/object:Gem::Dependency
-  name: faraday
+  name: relaton-index
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.7.0
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.7.0
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: relaton-iso-bib
   requirement: !ruby/object:Gem::Requirement