relaton-iec 1.14.2 → 1.14.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e377e8eaef84193231a109087a5d036e68819065c3256179533011f7291e205e
-  data.tar.gz: c4657c57400cc95b4707ffa0132865f793bf973fa9733683576ec88cef3fcb5d
+  metadata.gz: 4d00653dcd13da1fbfe251bd51366bcb0e038980a6a5598e8d06e90e93c25a5c
+  data.tar.gz: 1f70ddc26bc98c7a8da254e8eb7b31a28508c06763ebc3ab4de44dfc008b5465
 SHA512:
-  metadata.gz: 21f13b927c895e376e245d678e80d3185d6281b98f86e1cdb190905651685579286562b7d88786c52af70c4f1cc25433e3f6af41c290908ac8a46211eefb0602
-  data.tar.gz: 5f1c9e9e5a0068704a4619f0ec2fe9e958bc18d8c776822072317d3d252c2fe67774186be69547d9cc8b5ad22fb96c7c4e60777c33e0215c99f609c78b540997
+  metadata.gz: b4b1f3a22cfa10e630795cebe87ee63ef4e803f4ca276d1f94b37ab09e940d9abc430ab66fbd80e781c1dd8055dd948209f767449139d23bbc42ecbe4e4501cf
+  data.tar.gz: 9ad6a0084b3144ffde1c06237265ca2a6e73ae3762c4dc210ae199a70bdd931866f3db0dfdc1ebbbf17d3714942c33346dc9fb8ae1233910433ba6391bdd7281

data/Gemfile

@@ -4,3 +4,9 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in relaton_iec.gemspec
 gemspec
+
+gem "pry-byebug"
+gem "ruby-jing"
+gem "simplecov"
+gem "vcr"
+gem "webmock"

data/README.adoc

@@ -238,6 +238,28 @@ RelatonIec.urn_to_code "urn:iec:std:iec:60034-1:1969::csv:en-fr:plus:amd:1:1977:
 => ["IEC 60034-1:1969+AMD1:1977+AMD2:1979+AMD3:1980 CSV", "en-fr"]
 ----
 
+=== Fetch data
+
+This gem uses the https://bitbucket.org/sdo-hapi/api-specifications/wiki/Guidelines[Harmonized API] as a data source. The following options are available:
+- `iec-harmonized-all` - fetches all the documents using the API
+- `iec-harmonized-latest` - fetches the latest documents using the API
+
+The method `RelatonIec::DataFetcher.new(source, output: "data", format: "yaml").fetch` fetches IEC documents from the dataset and saves them to the `./data` folder in YAML format.
+Arguments:
+
+- `source` - data source. Possible values: `iec-harmonized-all`, `iec-harmonized-latest` (default `iec-harmonized-latest`).
+- `output` - folder to save documents (default './data').
+- `format` - the format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxxml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonIec::DataFetcher.new.fetch
+Started at: 2022-06-23 09:36:55 +0200
+Stopped at: 2022-06-23 09:36:58 +0200
+Done in: 752 sec.
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/relaton_iec/data_fetcher.rb

@@ -2,6 +2,7 @@ module RelatonIec
   class DataFetcher
     ENTRYPOINT = "https://api.iec.ch/harmonized/publications?size=100&sortBy=urn&page=".freeze
     CREDENTIAL = "https://api.iec.ch/oauth/client_credential/accesstoken?grant_type=client_credentials".freeze
+    LAST_CHANGE_FILE = "last_change.txt".freeze
 
     #
     # Initialize new instance.
@@ -15,10 +16,22 @@ module RelatonIec
       @format = format
       @ext = format.sub(/^bib/, "")
       @files = []
-      @index = Index.new "index.yaml"
+      # @index = Index.new "index.yaml"
+      @last_change = File.read(LAST_CHANGE_FILE, encoding: "UTF-8") if File.exist? LAST_CHANGE_FILE
+      @last_change_max = @last_change.to_s
       @all = source == "iec-harmonised-all"
     end
 
+    def last_change_max(date)
+      @last_change_max = date if @last_change_max < date
+    end
+
+    def save_last_change
+      return if @last_change_max.empty?
+
+      File.write LAST_CHANGE_FILE, @last_change_max, encoding: "UTF-8"
+    end
+
     #
     # Fetch data from IEC.
     #
@@ -26,14 +39,13 @@ module RelatonIec
       t1 = Time.now
       puts "Started at: #{t1}"
 
-      FileUtils.mkdir_p @output
       if @all
-        FileUtils.rm Dir[File.join(@output, "*.#{@ext}")]
-        @index.clear
+        FileUtils.rm_rf @output
       end
+      FileUtils.mkdir_p @output
       fetch_all
-      add_static_files_to_index
-      @index.save
+      create_index
+      save_last_change
 
       t2 = Time.now
       puts "Stopped at: #{t2}"
@@ -43,18 +55,29 @@ module RelatonIec
       warn e.backtrace.join("\n")
     end
 
+    def create_index
+      index = Relaton::Index.find_or_create :IEC, file: "index1.yaml"
+      index.remove_all
+      Dir["{#{@output},static}/*.yaml"].each do |file|
+        item = YAML.load_file file
+        id = item["docid"].detect { |i| i["primary"] }["id"]
+        index.add_or_update id, file
+      end
+      index.save
+    end
+
     #
     # Add static files to index.
     #
     # @return [void]
     #
-    def add_static_files_to_index
-      Dir["static/*.yaml"].each do |file|
-        pub = RelatonBib.parse_yaml File.read(file, encoding: "UTF-8")
-        pubid = RelatonBib.array(pub["docid"]).detect { |id| id["primary"] }["id"]
-        @index.add pubid, file
-      end
-    end
+    # def add_static_files_to_index
+    #   Dir["static/*.yaml"].each do |file|
+    #     pub = RelatonBib.parse_yaml File.read(file, encoding: "UTF-8")
+    #     pubid = RelatonBib.array(pub["docid"]).detect { |id| id["primary"] }["id"]
+    #     @index.add pubid, file
+    #   end
+    # end
 
     #
     # Fetch documents from IEC API.
@@ -102,8 +125,8 @@ module RelatonIec
     #
     def fetch_page(page)
       url = "#{ENTRYPOINT}#{page}"
-      if !@all && @index.last_change
-        url += "&lastChangeTimestampFrom=#{@index.last_change}"
+      if !@all && @last_change
+        url += "&lastChangeTimestampFrom=#{@last_change}"
       end
       uri = URI url
       req = Net::HTTP::Get.new uri
@@ -142,8 +165,9 @@ module RelatonIec
       if @files.include? file then warn "File #{file} exists."
       else
         @files << file
-        @index.add index_id(pub), file, pub["lastChangeTimestamp"]
+        # @index.add index_id(pub), file, pub["lastChangeTimestamp"]
       end
+      last_change_max pub["lastChangeTimestamp"]
       content = case @format
                 when "xml" then bib.to_xml bibdata: true
                 when "yaml", "yml" then bib.to_hash.to_yaml

data/lib/relaton_iec/data_parser.rb

@@ -177,14 +177,16 @@ module RelatonIec
     # @return [Array<RelatonBib::BibliographicDate>] dates
     #
     def date
-      date = []
-      date << create_date("published", @pub["releaseDate"]) if @pub["releaseDate"]
-      date << create_date("confirmed", @pub["confirmationDate"]) if @pub["confirmationDate"]
-      date
-    end
+      {
+        "published" => "publicationDate",
+        "stable-until" => "stabilityDate",
+        "confirmed" => "confirmationDate",
+        "obsoleted" => "dateOfWithdrawal",
+      }.reduce([]) do |a, (k, v)|
+        next a unless @pub[v]
 
-    def create_date(type, date)
-      RelatonBib::BibliographicDate.new(type: type, on: date)
+        a << RelatonBib::BibliographicDate.new(type: k, on: @pub[v])
+      end
     end
 
     #

data/lib/relaton_iec/hit.rb

@@ -22,5 +22,13 @@ module RelatonIec
     def part
       @part ||= hit[:code].match(/(?<=-)[\w-]+/)&.to_s
     end
+
+    def inspect
+      "<#{self.class}:#{format('%<id>#.14x', id: object_id << 1)} " \
+        "@text=\"#{@hit_collection&.text}\" " \
+        "@fetched=\"#{!@fetch.nil?}\" " \
+        "@fullIdentifier=\"#{@fetch&.shortref(nil, no_year: true)}\" " \
+        "@title=\"#{@hit[:code]}\">"
+    end
   end
 end

data/lib/relaton_iec/hit_collection.rb

@@ -14,7 +14,7 @@ module RelatonIec
     # @param year [String, nil]
     def initialize(ref, year = nil)
       super ref, year
-      @index = Index.new
+      @index = Relaton::Index.find_or_create :IEC, url: "#{Hit::GHURL}index1.zip" , file: "index1.yaml"
       @array = fetch_from_gh
     end
 
@@ -43,9 +43,11 @@ module RelatonIec
 
       ref = year && !/:\d{4}$/.match?(text) ? "#{text}:#{year}" : text
       reference = ref.sub(/^IEC\s(?=ISO\/IEC\sDIR)/, "")
-      @index.search(reference).map do |row|
+      @index.search do |row|
+        row[:id].include? reference
+      end.sort_by { |row| row[:id] }.map do |row|
         # pubid = row[:pubid].is_a?(Array) ? row[:pubid][0] : row[:pubid]
-        Hit.new({ code: row[:pubid], file: row[:file] }, self)
+        Hit.new({ code: row[:id], file: row[:file] }, self)
       end
     end
   end

data/lib/relaton_iec/version.rb

@@ -1,3 +1,3 @@
 module RelatonIec
-  VERSION = "1.14.2".freeze
+  VERSION = "1.14.3".freeze
 end

data/lib/relaton_iec.rb

@@ -3,6 +3,7 @@ require "net/http"
 require "nokogiri"
 require "zip"
 require "open-uri"
+require "relaton/index"
 require "relaton_iso_bib"
 require "relaton_iec/hit"
 require "relaton_iec/version"
@@ -10,7 +11,7 @@ require "relaton_iec/iec_bibliography"
 require "relaton_iec/iec_bibliographic_item"
 require "relaton_iec/xml_parser"
 require "relaton_iec/hash_converter"
-require "relaton_iec/index"
+# require "relaton_iec/index"
 require "relaton_iec/data_fetcher"
 require "relaton_iec/data_parser"
 

data/relaton_iec.gemspec

@@ -24,15 +24,11 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "ruby-jing"
-  spec.add_development_dependency "simplecov"
-  spec.add_development_dependency "vcr"
-  spec.add_development_dependency "webmock"
 
   spec.add_dependency "addressable"
+  spec.add_dependency "relaton-index", "~> 0.1.6"
   spec.add_dependency "relaton-iso-bib", "~> 1.14.0"
   spec.add_dependency "rubyzip"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-iec
 version: !ruby/object:Gem::Version
-  version: 1.14.2
+  version: 1.14.3
 platform: ruby
 authors:
 - Ribose Inc.
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-03-02 00:00:00.000000000 Z
+date: 2023-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -24,20 +24,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-- !ruby/object:Gem::Dependency
-  name: pry-byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -67,55 +53,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: ruby-jing
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: webmock
+  name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -123,19 +67,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: addressable
+  name: relaton-index
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.6
 - !ruby/object:Gem::Dependency
   name: relaton-iso-bib
   requirement: !ruby/object:Gem::Requirement
@@ -225,7 +169,7 @@ homepage: https://github.com/metanorma/relaton-iec
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -240,8 +184,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.4.9
+signing_key:
 specification_version: 4
 summary: 'RelatonIec: retrieve IEC Standards for bibliographic use using the IecBibliographicItem
   model'