relaton-iec 1.14.2 → 1.14.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e377e8eaef84193231a109087a5d036e68819065c3256179533011f7291e205e
4
- data.tar.gz: c4657c57400cc95b4707ffa0132865f793bf973fa9733683576ec88cef3fcb5d
3
+ metadata.gz: 4d00653dcd13da1fbfe251bd51366bcb0e038980a6a5598e8d06e90e93c25a5c
4
+ data.tar.gz: 1f70ddc26bc98c7a8da254e8eb7b31a28508c06763ebc3ab4de44dfc008b5465
5
5
  SHA512:
6
- metadata.gz: 21f13b927c895e376e245d678e80d3185d6281b98f86e1cdb190905651685579286562b7d88786c52af70c4f1cc25433e3f6af41c290908ac8a46211eefb0602
7
- data.tar.gz: 5f1c9e9e5a0068704a4619f0ec2fe9e958bc18d8c776822072317d3d252c2fe67774186be69547d9cc8b5ad22fb96c7c4e60777c33e0215c99f609c78b540997
6
+ metadata.gz: b4b1f3a22cfa10e630795cebe87ee63ef4e803f4ca276d1f94b37ab09e940d9abc430ab66fbd80e781c1dd8055dd948209f767449139d23bbc42ecbe4e4501cf
7
+ data.tar.gz: 9ad6a0084b3144ffde1c06237265ca2a6e73ae3762c4dc210ae199a70bdd931866f3db0dfdc1ebbbf17d3714942c33346dc9fb8ae1233910433ba6391bdd7281
data/Gemfile CHANGED
@@ -4,3 +4,9 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
4
4
 
5
5
  # Specify your gem's dependencies in relaton_iec.gemspec
6
6
  gemspec
7
+
8
+ gem "pry-byebug"
9
+ gem "ruby-jing"
10
+ gem "simplecov"
11
+ gem "vcr"
12
+ gem "webmock"
data/README.adoc CHANGED
@@ -238,6 +238,28 @@ RelatonIec.urn_to_code "urn:iec:std:iec:60034-1:1969::csv:en-fr:plus:amd:1:1977:
238
238
  => ["IEC 60034-1:1969+AMD1:1977+AMD2:1979+AMD3:1980 CSV", "en-fr"]
239
239
  ----
240
240
 
241
+ === Fetch data
242
+
243
+ This gem uses the https://bitbucket.org/sdo-hapi/api-specifications/wiki/Guidelines[Harmonized API] as a data source. The following options are available:
244
+ - `iec-harmonized-all` - fetches all the documents using the API
245
+ - `iec-harmonized-latest` - fetches the latest documents using the API
246
+
247
+ The method `RelatonIec::DataFetcher.new(source, output: "data", format: "yaml").fetch` fetches IEC documents from the dataset and saves them to the `./data` folder in YAML format.
248
+ Arguments:
249
+
250
+ - `source` - data source. Possible values: `iec-harmonized-all`, `iec-harmonized-latest` (default `iec-harmonized-latest`).
251
+ - `output` - folder to save documents (default './data').
252
+ - `format` - the format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxxml` (default `yaml`).
253
+
254
+ [source,ruby]
255
+ ----
256
+ RelatonIec::DataFetcher.new.fetch
257
+ Started at: 2022-06-23 09:36:55 +0200
258
+ Stopped at: 2022-06-23 09:36:58 +0200
259
+ Done in: 752 sec.
260
+ => nil
261
+ ----
262
+
241
263
  == Development
242
264
 
243
265
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -2,6 +2,7 @@ module RelatonIec
2
2
  class DataFetcher
3
3
  ENTRYPOINT = "https://api.iec.ch/harmonized/publications?size=100&sortBy=urn&page=".freeze
4
4
  CREDENTIAL = "https://api.iec.ch/oauth/client_credential/accesstoken?grant_type=client_credentials".freeze
5
+ LAST_CHANGE_FILE = "last_change.txt".freeze
5
6
 
6
7
  #
7
8
  # Initialize new instance.
@@ -15,10 +16,22 @@ module RelatonIec
15
16
  @format = format
16
17
  @ext = format.sub(/^bib/, "")
17
18
  @files = []
18
- @index = Index.new "index.yaml"
19
+ # @index = Index.new "index.yaml"
20
+ @last_change = File.read(LAST_CHANGE_FILE, encoding: "UTF-8") if File.exist? LAST_CHANGE_FILE
21
+ @last_change_max = @last_change.to_s
19
22
  @all = source == "iec-harmonised-all"
20
23
  end
21
24
 
25
+ def last_change_max(date)
26
+ @last_change_max = date if @last_change_max < date
27
+ end
28
+
29
+ def save_last_change
30
+ return if @last_change_max.empty?
31
+
32
+ File.write LAST_CHANGE_FILE, @last_change_max, encoding: "UTF-8"
33
+ end
34
+
22
35
  #
23
36
  # Fetch data from IEC.
24
37
  #
@@ -26,14 +39,13 @@ module RelatonIec
26
39
  t1 = Time.now
27
40
  puts "Started at: #{t1}"
28
41
 
29
- FileUtils.mkdir_p @output
30
42
  if @all
31
- FileUtils.rm Dir[File.join(@output, "*.#{@ext}")]
32
- @index.clear
43
+ FileUtils.rm_rf @output
33
44
  end
45
+ FileUtils.mkdir_p @output
34
46
  fetch_all
35
- add_static_files_to_index
36
- @index.save
47
+ create_index
48
+ save_last_change
37
49
 
38
50
  t2 = Time.now
39
51
  puts "Stopped at: #{t2}"
@@ -43,18 +55,29 @@ module RelatonIec
43
55
  warn e.backtrace.join("\n")
44
56
  end
45
57
 
58
+ def create_index
59
+ index = Relaton::Index.find_or_create :IEC, file: "index1.yaml"
60
+ index.remove_all
61
+ Dir["{#{@output},static}/*.yaml"].each do |file|
62
+ item = YAML.load_file file
63
+ id = item["docid"].detect { |i| i["primary"] }["id"]
64
+ index.add_or_update id, file
65
+ end
66
+ index.save
67
+ end
68
+
46
69
  #
47
70
  # Add static files to index.
48
71
  #
49
72
  # @return [void]
50
73
  #
51
- def add_static_files_to_index
52
- Dir["static/*.yaml"].each do |file|
53
- pub = RelatonBib.parse_yaml File.read(file, encoding: "UTF-8")
54
- pubid = RelatonBib.array(pub["docid"]).detect { |id| id["primary"] }["id"]
55
- @index.add pubid, file
56
- end
57
- end
74
+ # def add_static_files_to_index
75
+ # Dir["static/*.yaml"].each do |file|
76
+ # pub = RelatonBib.parse_yaml File.read(file, encoding: "UTF-8")
77
+ # pubid = RelatonBib.array(pub["docid"]).detect { |id| id["primary"] }["id"]
78
+ # @index.add pubid, file
79
+ # end
80
+ # end
58
81
 
59
82
  #
60
83
  # Fetch documents from IEC API.
@@ -102,8 +125,8 @@ module RelatonIec
102
125
  #
103
126
  def fetch_page(page)
104
127
  url = "#{ENTRYPOINT}#{page}"
105
- if !@all && @index.last_change
106
- url += "&lastChangeTimestampFrom=#{@index.last_change}"
128
+ if !@all && @last_change
129
+ url += "&lastChangeTimestampFrom=#{@last_change}"
107
130
  end
108
131
  uri = URI url
109
132
  req = Net::HTTP::Get.new uri
@@ -142,8 +165,9 @@ module RelatonIec
142
165
  if @files.include? file then warn "File #{file} exists."
143
166
  else
144
167
  @files << file
145
- @index.add index_id(pub), file, pub["lastChangeTimestamp"]
168
+ # @index.add index_id(pub), file, pub["lastChangeTimestamp"]
146
169
  end
170
+ last_change_max pub["lastChangeTimestamp"]
147
171
  content = case @format
148
172
  when "xml" then bib.to_xml bibdata: true
149
173
  when "yaml", "yml" then bib.to_hash.to_yaml
@@ -177,14 +177,16 @@ module RelatonIec
177
177
  # @return [Array<RelatonBib::BibliographicDate>] dates
178
178
  #
179
179
  def date
180
- date = []
181
- date << create_date("published", @pub["releaseDate"]) if @pub["releaseDate"]
182
- date << create_date("confirmed", @pub["confirmationDate"]) if @pub["confirmationDate"]
183
- date
184
- end
180
+ {
181
+ "published" => "publicationDate",
182
+ "stable-until" => "stabilityDate",
183
+ "confirmed" => "confirmationDate",
184
+ "obsoleted" => "dateOfWithdrawal",
185
+ }.reduce([]) do |a, (k, v)|
186
+ next a unless @pub[v]
185
187
 
186
- def create_date(type, date)
187
- RelatonBib::BibliographicDate.new(type: type, on: date)
188
+ a << RelatonBib::BibliographicDate.new(type: k, on: @pub[v])
189
+ end
188
190
  end
189
191
 
190
192
  #
@@ -22,5 +22,13 @@ module RelatonIec
22
22
  def part
23
23
  @part ||= hit[:code].match(/(?<=-)[\w-]+/)&.to_s
24
24
  end
25
+
26
+ def inspect
27
+ "<#{self.class}:#{format('%<id>#.14x', id: object_id << 1)} " \
28
+ "@text=\"#{@hit_collection&.text}\" " \
29
+ "@fetched=\"#{!@fetch.nil?}\" " \
30
+ "@fullIdentifier=\"#{@fetch&.shortref(nil, no_year: true)}\" " \
31
+ "@title=\"#{@hit[:code]}\">"
32
+ end
25
33
  end
26
34
  end
@@ -14,7 +14,7 @@ module RelatonIec
14
14
  # @param year [String, nil]
15
15
  def initialize(ref, year = nil)
16
16
  super ref, year
17
- @index = Index.new
17
+ @index = Relaton::Index.find_or_create :IEC, url: "#{Hit::GHURL}index1.zip" , file: "index1.yaml"
18
18
  @array = fetch_from_gh
19
19
  end
20
20
 
@@ -43,9 +43,11 @@ module RelatonIec
43
43
 
44
44
  ref = year && !/:\d{4}$/.match?(text) ? "#{text}:#{year}" : text
45
45
  reference = ref.sub(/^IEC\s(?=ISO\/IEC\sDIR)/, "")
46
- @index.search(reference).map do |row|
46
+ @index.search do |row|
47
+ row[:id].include? reference
48
+ end.sort_by { |row| row[:id] }.map do |row|
47
49
  # pubid = row[:pubid].is_a?(Array) ? row[:pubid][0] : row[:pubid]
48
- Hit.new({ code: row[:pubid], file: row[:file] }, self)
50
+ Hit.new({ code: row[:id], file: row[:file] }, self)
49
51
  end
50
52
  end
51
53
  end
@@ -1,3 +1,3 @@
1
1
  module RelatonIec
2
- VERSION = "1.14.2".freeze
2
+ VERSION = "1.14.3".freeze
3
3
  end
data/lib/relaton_iec.rb CHANGED
@@ -3,6 +3,7 @@ require "net/http"
3
3
  require "nokogiri"
4
4
  require "zip"
5
5
  require "open-uri"
6
+ require "relaton/index"
6
7
  require "relaton_iso_bib"
7
8
  require "relaton_iec/hit"
8
9
  require "relaton_iec/version"
@@ -10,7 +11,7 @@ require "relaton_iec/iec_bibliography"
10
11
  require "relaton_iec/iec_bibliographic_item"
11
12
  require "relaton_iec/xml_parser"
12
13
  require "relaton_iec/hash_converter"
13
- require "relaton_iec/index"
14
+ # require "relaton_iec/index"
14
15
  require "relaton_iec/data_fetcher"
15
16
  require "relaton_iec/data_parser"
16
17
 
data/relaton_iec.gemspec CHANGED
@@ -24,15 +24,11 @@ Gem::Specification.new do |spec|
24
24
  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
25
25
 
26
26
  spec.add_development_dependency "equivalent-xml", "~> 0.6"
27
- spec.add_development_dependency "pry-byebug"
28
27
  spec.add_development_dependency "rake", "~> 13.0"
29
28
  spec.add_development_dependency "rspec", "~> 3.0"
30
- spec.add_development_dependency "ruby-jing"
31
- spec.add_development_dependency "simplecov"
32
- spec.add_development_dependency "vcr"
33
- spec.add_development_dependency "webmock"
34
29
 
35
30
  spec.add_dependency "addressable"
31
+ spec.add_dependency "relaton-index", "~> 0.1.6"
36
32
  spec.add_dependency "relaton-iso-bib", "~> 1.14.0"
37
33
  spec.add_dependency "rubyzip"
38
34
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: relaton-iec
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.14.2
4
+ version: 1.14.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-03-02 00:00:00.000000000 Z
11
+ date: 2023-05-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: equivalent-xml
@@ -24,20 +24,6 @@ dependencies:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0.6'
27
- - !ruby/object:Gem::Dependency
28
- name: pry-byebug
29
- requirement: !ruby/object:Gem::Requirement
30
- requirements:
31
- - - ">="
32
- - !ruby/object:Gem::Version
33
- version: '0'
34
- type: :development
35
- prerelease: false
36
- version_requirements: !ruby/object:Gem::Requirement
37
- requirements:
38
- - - ">="
39
- - !ruby/object:Gem::Version
40
- version: '0'
41
27
  - !ruby/object:Gem::Dependency
42
28
  name: rake
43
29
  requirement: !ruby/object:Gem::Requirement
@@ -67,55 +53,13 @@ dependencies:
67
53
  - !ruby/object:Gem::Version
68
54
  version: '3.0'
69
55
  - !ruby/object:Gem::Dependency
70
- name: ruby-jing
71
- requirement: !ruby/object:Gem::Requirement
72
- requirements:
73
- - - ">="
74
- - !ruby/object:Gem::Version
75
- version: '0'
76
- type: :development
77
- prerelease: false
78
- version_requirements: !ruby/object:Gem::Requirement
79
- requirements:
80
- - - ">="
81
- - !ruby/object:Gem::Version
82
- version: '0'
83
- - !ruby/object:Gem::Dependency
84
- name: simplecov
85
- requirement: !ruby/object:Gem::Requirement
86
- requirements:
87
- - - ">="
88
- - !ruby/object:Gem::Version
89
- version: '0'
90
- type: :development
91
- prerelease: false
92
- version_requirements: !ruby/object:Gem::Requirement
93
- requirements:
94
- - - ">="
95
- - !ruby/object:Gem::Version
96
- version: '0'
97
- - !ruby/object:Gem::Dependency
98
- name: vcr
99
- requirement: !ruby/object:Gem::Requirement
100
- requirements:
101
- - - ">="
102
- - !ruby/object:Gem::Version
103
- version: '0'
104
- type: :development
105
- prerelease: false
106
- version_requirements: !ruby/object:Gem::Requirement
107
- requirements:
108
- - - ">="
109
- - !ruby/object:Gem::Version
110
- version: '0'
111
- - !ruby/object:Gem::Dependency
112
- name: webmock
56
+ name: addressable
113
57
  requirement: !ruby/object:Gem::Requirement
114
58
  requirements:
115
59
  - - ">="
116
60
  - !ruby/object:Gem::Version
117
61
  version: '0'
118
- type: :development
62
+ type: :runtime
119
63
  prerelease: false
120
64
  version_requirements: !ruby/object:Gem::Requirement
121
65
  requirements:
@@ -123,19 +67,19 @@ dependencies:
123
67
  - !ruby/object:Gem::Version
124
68
  version: '0'
125
69
  - !ruby/object:Gem::Dependency
126
- name: addressable
70
+ name: relaton-index
127
71
  requirement: !ruby/object:Gem::Requirement
128
72
  requirements:
129
- - - ">="
73
+ - - "~>"
130
74
  - !ruby/object:Gem::Version
131
- version: '0'
75
+ version: 0.1.6
132
76
  type: :runtime
133
77
  prerelease: false
134
78
  version_requirements: !ruby/object:Gem::Requirement
135
79
  requirements:
136
- - - ">="
80
+ - - "~>"
137
81
  - !ruby/object:Gem::Version
138
- version: '0'
82
+ version: 0.1.6
139
83
  - !ruby/object:Gem::Dependency
140
84
  name: relaton-iso-bib
141
85
  requirement: !ruby/object:Gem::Requirement
@@ -225,7 +169,7 @@ homepage: https://github.com/metanorma/relaton-iec
225
169
  licenses:
226
170
  - MIT
227
171
  metadata: {}
228
- post_install_message:
172
+ post_install_message:
229
173
  rdoc_options: []
230
174
  require_paths:
231
175
  - lib
@@ -240,8 +184,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
240
184
  - !ruby/object:Gem::Version
241
185
  version: '0'
242
186
  requirements: []
243
- rubygems_version: 3.1.6
244
- signing_key:
187
+ rubygems_version: 3.4.9
188
+ signing_key:
245
189
  specification_version: 4
246
190
  summary: 'RelatonIec: retrieve IEC Standards for bibliographic use using the IecBibliographicItem
247
191
  model'