relaton-bipm 1.11.9 → 1.12.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e11a60de824e9077a64e3b8ac31a578a9c5255a41ea8175d61b42186b6ef9eb6
4
- data.tar.gz: a179822bd35ada0812119e508b756c5a4edba82bf34352d2ba9c4fff4a9b4be6
3
+ metadata.gz: 34c3cc924311be1a1be33e18480b8521f168e1c72a46976045e9ccba69c38459
4
+ data.tar.gz: 4d929d15878f08fd5b696ecbc7fc3b0e5e94ad05e245c5fbc74125cc77ae0b8e
5
5
  SHA512:
6
- metadata.gz: bd9b8ced31703af3b6ceedefc793da7f1d009750f2fc8c75292690244fd3b2c426ea1d6e39ff358425031fa35d96db4db81fa48c03766448186fd2e990fe7827
7
- data.tar.gz: 5f719b30f70141397bffda7d72e1add612099b05cc36daef4fcaa3143c2513a7a2750d39c084a92f30c1fe70438fa5e651e8973feb90d60c97d50bede3fd141d
6
+ metadata.gz: 020f3459446a2a48e0d571933811888bfe4fbcd91263c482609de7a34a98df38a27fe30af2dfedaba79dbd796b9cd161049573fc2209093b73c78ead725f0389
7
+ data.tar.gz: f638c858d2384822f59dbe04d39fa0ab6f9cb9bbd42fffb2e9e57c917a5f8d64782ff7176abe98a3968b73eca59be0ed95d0822be32349c549c9b2d8b6213db9
data/.rubocop.yml CHANGED
@@ -7,6 +7,6 @@ require: rubocop-rails
7
7
  inherit_from:
8
8
  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
9
9
  AllCops:
10
- TargetRubyVersion: 2.5
10
+ TargetRubyVersion: 2.6
11
11
  Rails:
12
12
  Enabled: false
data/README.adoc CHANGED
@@ -259,6 +259,34 @@ RelatonBipm::BipmBibliographicItem.from_hash hash
259
259
  ...
260
260
  ----
261
261
 
262
+ === Fetch data
263
+
264
+ This gem uses following datasets as data sources:
265
+ - `bipm-data-outcomes` - looking for local directory with the repository https://github.com/metanorma/bipm-data-outcomes
266
+ - `si-brochute` - looking for local directory with the repository https://github.com/metanorma/bipm-si-brochure
267
+
268
+ The method `RelatonBipm::DataFetcher.fetch(sourece, output: "data", format: "yaml")` fetches all the documents from the datast and save them to the `./data` folder in YAML format.
269
+ Arguments:
270
+
271
+ - `source` - name of the source dataset
272
+ - `output` - folder to save documents (default './data').
273
+ - `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml`, `bibxxml` (default `yaml`).
274
+
275
+ [source,ruby]
276
+ ----
277
+ RelatonBipm::DataFetcher.fetch "bipm-data-outcomes"
278
+ Started at: 2022-06-23 09:36:55 +0200
279
+ Stopped at: 2022-06-23 09:36:58 +0200
280
+ Done in: 2 sec.
281
+ => nil
282
+
283
+ RelatonBipm::DataFetcher.fetch "si-brochure"
284
+ Started at: 2022-06-23 09:37:12 +0200
285
+ Stopped at: 2022-06-23 09:37:12 +0200
286
+ Done in: 0 sec.
287
+ => nil
288
+ ----
289
+
262
290
  == Development
263
291
 
264
292
  After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
data/grammars/biblio.rng CHANGED
@@ -614,12 +614,103 @@
614
614
  <optional>
615
615
  <ref name="fetched"/>
616
616
  </optional>
617
- <choice>
618
- <oneOrMore>
619
- <ref name="btitle"/>
620
- </oneOrMore>
617
+ <optional>
621
618
  <ref name="formattedref"/>
622
- </choice>
619
+ </optional>
620
+ <oneOrMore>
621
+ <ref name="btitle"/>
622
+ </oneOrMore>
623
+ <zeroOrMore>
624
+ <ref name="bsource"/>
625
+ </zeroOrMore>
626
+ <oneOrMore>
627
+ <ref name="docidentifier"/>
628
+ </oneOrMore>
629
+ <optional>
630
+ <ref name="docnumber"/>
631
+ </optional>
632
+ <zeroOrMore>
633
+ <ref name="bdate"/>
634
+ </zeroOrMore>
635
+ <zeroOrMore>
636
+ <ref name="contributor"/>
637
+ </zeroOrMore>
638
+ <optional>
639
+ <ref name="edition"/>
640
+ </optional>
641
+ <zeroOrMore>
642
+ <ref name="version"/>
643
+ </zeroOrMore>
644
+ <zeroOrMore>
645
+ <ref name="biblionote"/>
646
+ </zeroOrMore>
647
+ <zeroOrMore>
648
+ <ref name="language"/>
649
+ </zeroOrMore>
650
+ <zeroOrMore>
651
+ <ref name="script"/>
652
+ </zeroOrMore>
653
+ <zeroOrMore>
654
+ <ref name="bibabstract"/>
655
+ </zeroOrMore>
656
+ <optional>
657
+ <ref name="status"/>
658
+ </optional>
659
+ <zeroOrMore>
660
+ <ref name="copyright"/>
661
+ </zeroOrMore>
662
+ <zeroOrMore>
663
+ <ref name="docrelation"/>
664
+ </zeroOrMore>
665
+ <zeroOrMore>
666
+ <ref name="series"/>
667
+ </zeroOrMore>
668
+ <optional>
669
+ <ref name="medium"/>
670
+ </optional>
671
+ <zeroOrMore>
672
+ <ref name="bplace"/>
673
+ </zeroOrMore>
674
+ <zeroOrMore>
675
+ <ref name="bprice"/>
676
+ </zeroOrMore>
677
+ <zeroOrMore>
678
+ <ref name="extent"/>
679
+ </zeroOrMore>
680
+ <optional>
681
+ <ref name="bibliographic_size"/>
682
+ </optional>
683
+ <zeroOrMore>
684
+ <ref name="accesslocation"/>
685
+ </zeroOrMore>
686
+ <zeroOrMore>
687
+ <ref name="license"/>
688
+ </zeroOrMore>
689
+ <zeroOrMore>
690
+ <ref name="bclassification"/>
691
+ </zeroOrMore>
692
+ <zeroOrMore>
693
+ <ref name="bkeyword"/>
694
+ </zeroOrMore>
695
+ <optional>
696
+ <ref name="validity"/>
697
+ </optional>
698
+ </define>
699
+ <define name="ReducedBibliographicItem">
700
+ <optional>
701
+ <attribute name="type">
702
+ <ref name="BibItemType"/>
703
+ </attribute>
704
+ </optional>
705
+ <optional>
706
+ <ref name="fetched"/>
707
+ </optional>
708
+ <optional>
709
+ <ref name="formattedref"/>
710
+ </optional>
711
+ <zeroOrMore>
712
+ <ref name="btitle"/>
713
+ </zeroOrMore>
623
714
  <zeroOrMore>
624
715
  <ref name="bsource"/>
625
716
  </zeroOrMore>
@@ -638,9 +729,9 @@
638
729
  <optional>
639
730
  <ref name="edition"/>
640
731
  </optional>
641
- <optional>
732
+ <zeroOrMore>
642
733
  <ref name="version"/>
643
- </optional>
734
+ </zeroOrMore>
644
735
  <zeroOrMore>
645
736
  <ref name="biblionote"/>
646
737
  </zeroOrMore>
@@ -833,6 +924,12 @@
833
924
  <data type="boolean"/>
834
925
  </attribute>
835
926
  </optional>
927
+ <optional>
928
+ <attribute name="language"/>
929
+ </optional>
930
+ <optional>
931
+ <attribute name="script"/>
932
+ </optional>
836
933
  <text/>
837
934
  </element>
838
935
  </define>
@@ -986,36 +1083,34 @@
986
1083
  <ref name="SeriesType"/>
987
1084
  </attribute>
988
1085
  </optional>
989
- <choice>
1086
+ <optional>
990
1087
  <ref name="formattedref"/>
991
- <group>
992
- <ref name="btitle"/>
993
- <optional>
994
- <ref name="bplace"/>
995
- </optional>
996
- <optional>
997
- <ref name="seriesorganization"/>
998
- </optional>
999
- <optional>
1000
- <ref name="abbreviation"/>
1001
- </optional>
1002
- <optional>
1003
- <ref name="seriesfrom"/>
1004
- </optional>
1005
- <optional>
1006
- <ref name="seriesto"/>
1007
- </optional>
1008
- <optional>
1009
- <ref name="seriesnumber"/>
1010
- </optional>
1011
- <optional>
1012
- <ref name="seriespartnumber"/>
1013
- </optional>
1014
- <optional>
1015
- <ref name="seriesrun"/>
1016
- </optional>
1017
- </group>
1018
- </choice>
1088
+ </optional>
1089
+ <ref name="btitle"/>
1090
+ <optional>
1091
+ <ref name="bplace"/>
1092
+ </optional>
1093
+ <optional>
1094
+ <ref name="seriesorganization"/>
1095
+ </optional>
1096
+ <optional>
1097
+ <ref name="abbreviation"/>
1098
+ </optional>
1099
+ <optional>
1100
+ <ref name="seriesfrom"/>
1101
+ </optional>
1102
+ <optional>
1103
+ <ref name="seriesto"/>
1104
+ </optional>
1105
+ <optional>
1106
+ <ref name="seriesnumber"/>
1107
+ </optional>
1108
+ <optional>
1109
+ <ref name="seriespartnumber"/>
1110
+ </optional>
1111
+ <optional>
1112
+ <ref name="seriesrun"/>
1113
+ </optional>
1019
1114
  </element>
1020
1115
  </define>
1021
1116
  <define name="SeriesType">
@@ -1174,7 +1269,7 @@
1174
1269
  </element>
1175
1270
  </optional>
1176
1271
  <element name="bibitem">
1177
- <ref name="BibliographicItem"/>
1272
+ <ref name="ReducedBibliographicItem"/>
1178
1273
  </element>
1179
1274
  <choice>
1180
1275
  <zeroOrMore>
@@ -1199,9 +1294,9 @@
1199
1294
  <optional>
1200
1295
  <ref name="revision-date"/>
1201
1296
  </optional>
1202
- <zeroOrMore>
1297
+ <optional>
1203
1298
  <ref name="draft"/>
1204
- </zeroOrMore>
1299
+ </optional>
1205
1300
  </element>
1206
1301
  </define>
1207
1302
  <define name="vedition">
data/grammars/bipm.rng CHANGED
@@ -198,6 +198,9 @@
198
198
  <optional>
199
199
  <attribute name="start"/>
200
200
  </optional>
201
+ <optional>
202
+ <ref name="tname"/>
203
+ </optional>
201
204
  <oneOrMore>
202
205
  <ref name="li"/>
203
206
  </oneOrMore>
data/grammars/isodoc.rng CHANGED
@@ -32,6 +32,43 @@
32
32
  <ref name="DocumentType"/>
33
33
  </element>
34
34
  </define>
35
+ <define name="admonition">
36
+ <element name="admonition">
37
+ <attribute name="type">
38
+ <ref name="AdmonitionType"/>
39
+ </attribute>
40
+ <optional>
41
+ <attribute name="class"/>
42
+ </optional>
43
+ <attribute name="id">
44
+ <data type="ID"/>
45
+ </attribute>
46
+ <optional>
47
+ <attribute name="uri">
48
+ <data type="anyURI"/>
49
+ </attribute>
50
+ </optional>
51
+ <optional>
52
+ <attribute name="coverpage">
53
+ <data type="boolean"/>
54
+ </attribute>
55
+ </optional>
56
+ <optional>
57
+ <attribute name="notag">
58
+ <data type="boolean"/>
59
+ </attribute>
60
+ </optional>
61
+ <optional>
62
+ <ref name="tname"/>
63
+ </optional>
64
+ <zeroOrMore>
65
+ <ref name="paragraph-with-footnote"/>
66
+ </zeroOrMore>
67
+ <zeroOrMore>
68
+ <ref name="note"/>
69
+ </zeroOrMore>
70
+ </element>
71
+ </define>
35
72
  <define name="index">
36
73
  <element name="index">
37
74
  <optional>
@@ -79,6 +116,11 @@
79
116
  <data type="boolean"/>
80
117
  </attribute>
81
118
  </optional>
119
+ <optional>
120
+ <attribute name="suppress_identifier">
121
+ <data type="boolean"/>
122
+ </attribute>
123
+ </optional>
82
124
  <ref name="BibliographicItem"/>
83
125
  </element>
84
126
  </define>
@@ -245,6 +287,9 @@
245
287
  <ref name="MultilingualRenderingType"/>
246
288
  </attribute>
247
289
  </optional>
290
+ <optional>
291
+ <ref name="tname"/>
292
+ </optional>
248
293
  <oneOrMore>
249
294
  <ref name="ul_li"/>
250
295
  </oneOrMore>
@@ -287,6 +332,9 @@
287
332
  </choice>
288
333
  </attribute>
289
334
  </optional>
335
+ <optional>
336
+ <ref name="tname"/>
337
+ </optional>
290
338
  <oneOrMore>
291
339
  <ref name="li"/>
292
340
  </oneOrMore>
@@ -323,6 +371,9 @@
323
371
  <ref name="MultilingualRenderingType"/>
324
372
  </attribute>
325
373
  </optional>
374
+ <optional>
375
+ <ref name="tname"/>
376
+ </optional>
326
377
  <oneOrMore>
327
378
  <ref name="dt"/>
328
379
  <ref name="dd"/>
@@ -657,6 +708,9 @@
657
708
  <optional>
658
709
  <attribute name="tag"/>
659
710
  </optional>
711
+ <optional>
712
+ <attribute name="type"/>
713
+ </optional>
660
714
  <optional>
661
715
  <attribute name="multilingual-rendering">
662
716
  <ref name="MultilingualRenderingType"/>
@@ -692,6 +746,9 @@
692
746
  <optional>
693
747
  <attribute name="tag"/>
694
748
  </optional>
749
+ <optional>
750
+ <attribute name="type"/>
751
+ </optional>
695
752
  <optional>
696
753
  <attribute name="multilingual-rendering">
697
754
  <ref name="MultilingualRenderingType"/>
@@ -851,6 +908,16 @@
851
908
  <ref name="MultilingualRenderingType"/>
852
909
  </attribute>
853
910
  </optional>
911
+ <optional>
912
+ <attribute name="coverpage">
913
+ <data type="boolean"/>
914
+ </attribute>
915
+ </optional>
916
+ <optional>
917
+ <attribute name="notag">
918
+ <data type="boolean"/>
919
+ </attribute>
920
+ </optional>
854
921
  <oneOrMore>
855
922
  <choice>
856
923
  <ref name="paragraph"/>
@@ -1178,6 +1245,7 @@
1178
1245
  <ref name="concept"/>
1179
1246
  <ref name="add"/>
1180
1247
  <ref name="del"/>
1248
+ <ref name="span"/>
1181
1249
  </choice>
1182
1250
  </define>
1183
1251
  <define name="add">
@@ -1204,6 +1272,14 @@
1204
1272
  </choice>
1205
1273
  </element>
1206
1274
  </define>
1275
+ <define name="span">
1276
+ <element name="span">
1277
+ <attribute name="class"/>
1278
+ <oneOrMore>
1279
+ <ref name="TextElement"/>
1280
+ </oneOrMore>
1281
+ </element>
1282
+ </define>
1207
1283
  <define name="concept">
1208
1284
  <element name="concept">
1209
1285
  <optional>
@@ -1994,6 +2070,7 @@
1994
2070
  <value>compare</value>
1995
2071
  <value>contrast</value>
1996
2072
  <value>see</value>
2073
+ <value>seealso</value>
1997
2074
  </choice>
1998
2075
  </define>
1999
2076
  <define name="deprecates">
@@ -4,13 +4,12 @@ module RelatonBipm
4
4
  class BipmBibliography
5
5
  GH_ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-bipm/master/".freeze
6
6
  IOP_DOMAIN = "https://iopscience.iop.org".freeze
7
- SUBSTITUTES = {
7
+ TRANSLATIONS = {
8
8
  "Déclaration" => "Declaration",
9
9
  "Réunion" => "Meeting",
10
10
  "Recommandation" => "Recommendation",
11
11
  "Résolution" => "Resolution",
12
12
  "Décision" => "Decision",
13
- "CCDS" => "CCTF",
14
13
  }.freeze
15
14
 
16
15
  class << self
@@ -51,7 +50,8 @@ module RelatonBipm
51
50
  rf = ref.sub(/(?:(\d{1,2})\s)?\(?(\d{4})(?!-)\)?/) do
52
51
  "#{$2}-#{$1.to_s.rjust(2, '0')}"
53
52
  end
54
- SUBSTITUTES.each { |fr, en| rf.sub! fr, en }
53
+ rf.sub!("CCDS", "CCTF")
54
+ TRANSLATIONS.each { |fr, en| rf.sub! fr, en }
55
55
  path = Index.new.search rf
56
56
  return unless path
57
57
 
@@ -60,7 +60,7 @@ module RelatonBipm
60
60
  check_response resp
61
61
  return unless resp.code == "200"
62
62
 
63
- yaml = RelatonBipm.parse_yaml resp.body, [Date]
63
+ yaml = RelatonBib.parse_yaml resp.body, [Date]
64
64
  bib_hash = HashConverter.hash_to_bib yaml
65
65
  BipmBibliographicItem.new(**bib_hash)
66
66
  end
@@ -1,9 +1,332 @@
1
1
  module RelatonBipm
2
2
  class DataFetcher
3
+ #
4
+ # Initialize fetcher
5
+ #
6
+ # @param [String] output output directory to save files
7
+ # @param [String] format format of output files (xml, yaml, bibxml)
8
+ #
3
9
  def initialize(output, format)
4
10
  @output = output
5
11
  @format = format
6
12
  @ext = format.sub(/^bib/, "")
13
+ @files = []
14
+ @index_path = "index.yaml"
15
+ @index = File.exist?(@index_path) ? YAML.load_file(@index_path) : {}
16
+ end
17
+
18
+ #
19
+ # Initialize fetcher and run fetching
20
+ #
21
+ # @param [String] source Source name
22
+ # @param [Strin] output directory to save files, default: "data"
23
+ # @param [Strin] format format of output files (xml, yaml, bibxml), default: yaml
24
+ #
25
+ def self.fetch(source, output: "data", format: "yaml")
26
+ t1 = Time.now
27
+ puts "Started at: #{t1}"
28
+ FileUtils.mkdir_p output unless Dir.exist? output
29
+ new(output, format).fetch(source)
30
+ t2 = Time.now
31
+ puts "Stopped at: #{t2}"
32
+ puts "Done in: #{(t2 - t1).round} sec."
33
+ end
34
+
35
+ #
36
+ # Fetch bipm-data-outcomes or si-brochure
37
+ #
38
+ # @param [String] source Source name
39
+ #
40
+ def fetch(source)
41
+ case source
42
+ when "bipm-data-outcomes" then parse_bipm_data_outcomes
43
+ when "si-brochure" then parse_si_brochure
44
+ end
45
+ File.write @index_path, @index.to_yaml, encoding: "UTF-8"
46
+ end
47
+
48
+ #
49
+ # Parse BIPM meeting and write them to YAML files
50
+ #
51
+ def parse_bipm_data_outcomes
52
+ source_path = File.join "bipm-data-outcomes", "{cctf,cgpm,cipm}"
53
+ Dir[source_path].each { |body_dir| fetch_body(body_dir) }
54
+ end
55
+
56
+ #
57
+ # Parse SI brochure and write them to YAML files
58
+ #
59
+ def parse_si_brochure # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
60
+ Dir["bipm-si-brochure/site/documents/*.rxl"].each do |f|
61
+ docstd = Nokogiri::XML File.read f
62
+ doc = docstd.at "/bibdata"
63
+ hash1 = RelatonBipm::XMLParser.from_xml(doc.to_xml).to_hash
64
+ hash1["fetched"] = Date.today.to_s
65
+ hash1["docid"].detect { |id| id["type"] == "BIPM" }["primary"] = true
66
+ outfile = File.join @output, File.basename(f).sub(/(?:-(?:en|fr))?\.rxl$/, ".yaml")
67
+ @index[[hash1["docnumber"] || File.basename(outfile, ".yaml")]] = outfile
68
+ hash = if File.exist? outfile
69
+ warn_duplicate = false
70
+ hash2 = YAML.load_file outfile
71
+ deep_merge hash1, hash2
72
+ else
73
+ warn_duplicate = true
74
+ hash1
75
+ end
76
+ item = RelatonBipm::BipmBibliographicItem.from_hash(**hash)
77
+ write_file outfile, item, warn_duplicate: warn_duplicate
78
+ end
79
+ end
80
+
81
+ #
82
+ # Deep merge two hashes
83
+ #
84
+ # @param [Hash] hash1
85
+ # @param [Hash] hash2
86
+ #
87
+ # @return [Hash] Merged hash
88
+ #
89
+ def deep_merge(hash1, hash2) # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
90
+ hash1.merge(hash2) do |_, oldval, newval|
91
+ if oldval.is_a?(Hash) && newval.is_a?(Hash)
92
+ deep_merge(oldval, newval)
93
+ elsif oldval.is_a?(Array) && newval.is_a?(Array)
94
+ oldval | newval
95
+ else
96
+ newval || oldval
97
+ end
98
+ end
99
+ end
100
+
101
+ #
102
+ # Search for English meetings in the body directory
103
+ #
104
+ # @param [String] dir body directory
105
+ #
106
+ def fetch_body(dir)
107
+ body = dir.split("/").last.upcase
108
+ Dir[File.join(dir, "*-en")].each { |type_dir| fetch_type type_dir, body }
109
+ end
110
+
111
+ #
112
+ # Search for meetings
113
+ #
114
+ # @param [String] dir meeting directory
115
+ # @param [String] body name of body
116
+ #
117
+ def fetch_type(dir, body) # rubocop:disable Metrics/AbcSize
118
+ type = dir.split("/").last.split("-").first.sub(/s$/, "")
119
+ body_dir = File.join @output, body.downcase
120
+ Dir.mkdir body_dir unless Dir.exist? body_dir
121
+ outdir = File.join body_dir, type.downcase
122
+ Dir.mkdir outdir unless Dir.exist? outdir
123
+ Dir[File.join(dir, "*.{yml,yaml}")].each { |en_file| fetch_meeting en_file, body, type, outdir }
124
+ end
125
+
126
+ #
127
+ # Create and write BIPM meeting/resolution
128
+ #
129
+ # @param [String] en_file Path to English file
130
+ # @param [String] body Body name
131
+ # @param [String] type Type of Recommendation/Decision/Resolution
132
+ # @param [String] dir output directory
133
+ #
134
+ def fetch_meeting(en_file, body, type, dir) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
135
+ en = RelatonBib.parse_yaml File.read(en_file, encoding: "UTF-8"), [Date]
136
+ en_md = en["metadata"]
137
+ fr_file = en_file.sub "en", "fr"
138
+ fr = RelatonBib.parse_yaml File.read(fr_file, encoding: "UTF-8"), [Date]
139
+ fr_md = fr["metadata"]
140
+ gh_src = "https://raw.githubusercontent.com/metanorma/"
141
+ src_en = gh_src + en_file.split("/")[1..].insert(1, "main").join("/")
142
+ src_fr = gh_src + fr_file.split("/")[1..].insert(1, "main").join("/")
143
+ src = [{ type: "src", content: src_en }, { type: "src", content: src_fr }]
144
+
145
+ /^(?<num>\d+)(?:-_(?<part>\d+))?-\d{4}$/ =~ en_md["url"].split("/").last
146
+ id = "#{body} #{type.capitalize} #{num}"
147
+ file = "#{num}.yaml"
148
+ path = File.join dir, file
149
+ link = "https://raw.githubusercontent.com/relaton/relaton-data-bipm/master/#{path}"
150
+ hash = bibitem type: type, en: en_md, fr: fr_md, id: id, num: num, src: src, pdf: en["pdf"]
151
+ if @files.include?(path) && part
152
+ add_part hash, part
153
+ item = RelatonBipm::BipmBibliographicItem.new(**hash)
154
+ yaml = RelatonBib.parse_yaml(File.read(path, encoding: "UTF-8"), [Date])
155
+ has_part_item = RelatonBipm::BipmBibliographicItem.from_hash(yaml)
156
+ has_part_item.relation << RelatonBib::DocumentRelation.new(type: "partOf", bibitem: item)
157
+ write_file path, has_part_item, warn_duplicate: false
158
+ path = File.join dir, "#{num}-#{part}.yaml"
159
+ elsif part
160
+ hash[:title].each { |t| t[:content] = t[:content].sub(/\s\(.+\)$/, "") }
161
+ hash[:link] = [{ type: "src", content: link }]
162
+ h = bibitem type: type, en: en_md, fr: fr_md, id: id, num: num, src: src, pdf: en["pdf"]
163
+ add_part h, part
164
+ part_item = RelatonBipm::BipmBibliographicItem.new(**h)
165
+ part_item_path = File.join dir, "#{num}-#{part}.yaml"
166
+ write_file part_item_path, part_item
167
+ @index[[h[:docnumber]]] = part_item_path
168
+ hash[:relation] = [RelatonBib::DocumentRelation.new(type: "partOf", bibitem: part_item)]
169
+ item = RelatonBipm::BipmBibliographicItem.new(**hash)
170
+ else
171
+ item = RelatonBipm::BipmBibliographicItem.new(**hash)
172
+ end
173
+ write_file path, item
174
+ @index[[hash[:docnumber]]] = path
175
+ fetch_resolution body: body, en: en, fr: fr, dir: dir, src: src, num: num
176
+ end
177
+
178
+ #
179
+ # Parse BIPM resolutions and write them to YAML files
180
+ #
181
+ # @param [String] body body name
182
+ # @param [Hash] eng English metadata
183
+ # @param [Hash] frn French metadata
184
+ # @param [String] dir output directory
185
+ # @param [Array<Hash>] src links to bipm-data-outcomes
186
+ # @param [String] num number of meeting
187
+ #
188
+ def fetch_resolution(**args) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
189
+ args[:en]["resolutions"].each.with_index do |r, i| # rubocop:disable Metrics/BlockLength
190
+ hash = { fetched: Date.today.to_s, title: [], doctype: r["type"] }
191
+ hash[:title] << title(r["title"], "en") if r["title"]
192
+ fr_title = args[:fr]["resolutions"][i]["title"]
193
+ hash[:title] << title(fr_title, "fr") if fr_title
194
+ date = r["dates"].first.to_s
195
+ hash[:date] = [{ type: "published", on: date }]
196
+ num = r["identifier"].to_s.split("-").last
197
+ year = date.split("-").first
198
+ num = "0" if num == year
199
+ num_justed = num.rjust 2, "0"
200
+ type = r["type"].capitalize
201
+ id = "#{args[:body]} #{type}"
202
+ hash[:id] = "#{args[:body]}-#{type}-#{year}"
203
+ if num.to_i.positive?
204
+ id += " #{num}"
205
+ hash[:id] += "-#{num_justed}"
206
+ end
207
+ id += " (#{year})"
208
+ hash[:docid] = [
209
+ make_docid(id: id, type: "BIPM", primary: true),
210
+ make_docid(id: id, type: "BIPM", primary: true, language: "en", script: "Latn"),
211
+ id_fr(id),
212
+ ]
213
+ hash[:docnumber] = id
214
+ hash[:link] = [{ type: "src", content: r["url"] }] + args[:src]
215
+ hash[:link] << { type: "pdf", content: r["reference"] } if r["reference"]
216
+ hash[:language] = %w[en fr]
217
+ hash[:script] = ["Latn"]
218
+ hash[:contributor] = [{
219
+ entity: { url: "www.bipm.org", name: "Bureau International des Poids et Mesures", abbreviation: "BIPM" },
220
+ role: [{ type: "publisher" }],
221
+ }]
222
+ hash[:structuredidentifier] = RelatonBipm::StructuredIdentifier.new docnumber: num
223
+ item = RelatonBipm::BipmBibliographicItem.new(**hash)
224
+ file = year
225
+ file += "-#{num_justed}" if num.size < 4
226
+ file += ".yaml"
227
+ out_dir = File.join args[:dir], r["type"].downcase
228
+ Dir.mkdir out_dir unless Dir.exist? out_dir
229
+ path = File.join out_dir, file
230
+ write_file path, item
231
+ @index[["#{args[:body]} #{type} #{year}-#{num_justed}", "#{args[:body]} #{type} #{args[:num]}-#{num_justed}"]] = path
232
+ end
233
+ end
234
+
235
+ def title(content, language)
236
+ { content: content, language: language, script: "Latn" }
237
+ end
238
+
239
+ #
240
+ # Add part to ID and structured identifier
241
+ #
242
+ # @param [Hash] hash Hash of BIPM meeting
243
+ # @param [String] session number of meeting
244
+ #
245
+ def add_part(hash, part)
246
+ hash[:id] += "-#{part}"
247
+ hash[:docnumber] += "-#{part}"
248
+ id = hash[:docid][0].instance_variable_get(:@id)
249
+ id += "-#{part}"
250
+ hash[:docid][0].instance_variable_set(:@id, id)
251
+ hash[:structuredidentifier].instance_variable_set :@part, part
252
+ end
253
+
254
+ #
255
+ # Create hash from BIPM meeting/resolution
256
+ #
257
+ # @param [Hash] **args Hash of arguments
258
+ # @option args [String] :type Type of meeting/resolution
259
+ # @option args [Hash] :en Hash of English metadata
260
+ # @option args [Hash] :fr Hash of French metadata
261
+ # @option args [String] :id ID of meeting/resolution
262
+ # @option args [String] :num Number of meeting/resolution
263
+ # @option args [Array<Hash>] :src Array of links to bipm-data-outcomes
264
+ # @option args [String] :pdf link to PDF
265
+ #
266
+ # @return [Hash] Hash of BIPM meeting/resolution
267
+ #
268
+ def bibitem(**args) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity
269
+ hash = { title: [], doctype: args[:type], fetched: Date.today.to_s }
270
+ hash[:title] << title(args[:en]["title"], "en") if args[:en]["title"]
271
+ hash[:title] << title(args[:fr]["title"], "fr") if args[:fr]["title"]
272
+ hash[:date] = [{ type: "published", on: args[:en]["date"] }]
273
+ hash[:docid] = [
274
+ make_docid(id: args[:id], type: "BIPM", primary: true),
275
+ make_docid(id: args[:id], type: "BIPM", primary: true, language: "en", script: "Latn"),
276
+ id_fr(args[:id]),
277
+ ]
278
+ hash[:id] = args[:id].gsub " ", "-"
279
+ hash[:docnumber] = args[:id]
280
+ hash[:link] = [{ type: "src", content: args[:en]["url"] }]
281
+ hash[:link] << { type: "pdf", content: args[:pdf] } if args[:pdf]
282
+ hash[:link] += args[:src] if args[:src]&.any?
283
+ hash[:language] = %w[en fr]
284
+ hash[:script] = ["Latn"]
285
+ hash[:contributor] = [{
286
+ entity: { url: "www.bipm.org", name: "Bureau International des Poids et Mesures", abbreviation: "BIPM" },
287
+ role: [{ type: "publisher" }],
288
+ }]
289
+ hash[:structuredidentifier] = RelatonBipm::StructuredIdentifier.new docnumber: args[:num]
290
+ hash
291
+ end
292
+
293
+ def id_fr(en_id)
294
+ tr = BipmBibliography::TRANSLATIONS.detect { |_, v| en_id.include? v }
295
+ id = en_id.sub tr[1], tr[0]
296
+ make_docid(id: id, type: "BIPM", primary: true, language: "fr", script: "Latn")
297
+ end
298
+
299
+ #
300
+ # Create doucment ID
301
+ #
302
+ # @param [String] id ID of document
303
+ # @param [String] type Type of document
304
+ # @param [Boolean] primary Primary document
305
+ # @param [String] language Language of document
306
+ # @param [String] script Script of document
307
+ #
308
+ # @return [RelatonBib::DocumentIdentifier] Document ID
309
+ #
310
+ def make_docid(**args)
311
+ RelatonBib::DocumentIdentifier.new(**args)
312
+ end
313
+
314
+ #
315
+ # Save document to file
316
+ #
317
+ # @param [String] path Path to file
318
+ # @param [RelatonBipm::BipmBibliographicItem] item document to save
319
+ # @param [Boolean, nil] warn_duplicate Warn if document already exists
320
+ #
321
+ # @return [<Type>] <description>
322
+ #
323
+ def write_file(path, item, warn_duplicate: true)
324
+ if @files.include?(path)
325
+ warn "File #{path} already exists" if warn_duplicate
326
+ else
327
+ @files << path
328
+ end
329
+ File.write path, item.to_hash.to_yaml, encoding: "UTF-8"
7
330
  end
8
331
  end
9
332
  end
@@ -9,7 +9,7 @@ module RelatonBipm
9
9
  # @param args [Hash]
10
10
  # @param nested [TrueClass, FalseClass]
11
11
  # @return [Hash]
12
- def hash_to_bib(args, nested = false)
12
+ def hash_to_bib(args)
13
13
  ret = super
14
14
  return if ret.nil?
15
15
 
@@ -41,7 +41,7 @@ module RelatonBipm
41
41
  def read_index_file
42
42
  return if !File.exist?(path) || File.ctime(path).to_date < Date.today
43
43
 
44
- @index = RelatonBipm.parse_yaml File.read(path, encoding: "UTF-8")
44
+ @index = RelatonBib.parse_yaml File.read(path, encoding: "UTF-8")
45
45
  end
46
46
 
47
47
  #
@@ -61,7 +61,7 @@ module RelatonBipm
61
61
  def get_index_from_gh # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
62
62
  resp = Zip::InputStream.new URI("#{BipmBibliography::GH_ENDPOINT}index.zip").open
63
63
  zip = resp.get_next_entry
64
- @index = RelatonBipm.parse_yaml zip.get_input_stream.read
64
+ @index = RelatonBib.parse_yaml zip.get_input_stream.read
65
65
  save_index_file
66
66
  end
67
67
  end
@@ -9,6 +9,7 @@ module RelatonBipm
9
9
  @prefix = "BIPM"
10
10
  @defaultprefix = %r{^(?:BIPM|CCTF|CCDS|CGPM|CIPM)(?!\w)}
11
11
  @idtype = "BIPM"
12
+ @datasets = %w[bipm-data-outcomes si-brochure]
12
13
  end
13
14
 
14
15
  # @param code [String]
@@ -19,6 +20,19 @@ module RelatonBipm
19
20
  ::RelatonBipm::BipmBibliography.get(code, date, opts)
20
21
  end
21
22
 
23
+ #
24
+ # Fetch all the documents from https://github.com/metanorma/bipm-data-outcomes
25
+ # or https://github.com/metanorma/bipm-si-brochure
26
+ #
27
+ # @param [String] source source name
28
+ # @param [Hash] opts
29
+ # @option opts [String] :output directory to output documents
30
+ # @option opts [String] :format
31
+ #
32
+ def fetch_data(source, opts)
33
+ DataFetcher.fetch(source, **opts)
34
+ end
35
+
22
36
  # @param xml [String]
23
37
  # @return [RelatonBipm::BipmBibliographicItem]
24
38
  def from_xml(xml)
@@ -1,3 +1,3 @@
1
1
  module RelatonBipm
2
- VERSION = "1.11.9".freeze
2
+ VERSION = "1.12.0".freeze
3
3
  end
data/lib/relaton_bipm.rb CHANGED
@@ -14,6 +14,7 @@ require "relaton_bipm/bipm_bibliography"
14
14
  require "relaton_bipm/hash_converter"
15
15
  require "relaton_bipm/xml_parser"
16
16
  require "relaton_bipm/index"
17
+ require "relaton_bipm/data_fetcher"
17
18
 
18
19
  module RelatonBipm
19
20
  class Error < StandardError; end
@@ -27,21 +28,5 @@ module RelatonBipm
27
28
  Digest::MD5.hexdigest grammars
28
29
  end
29
30
 
30
- #
31
- # Parse yaml content
32
- #
33
- # @param [String] yaml content
34
- #
35
- # @return [Hash] data
36
- #
37
- def parse_yaml(yaml, classes = [])
38
- # Newer versions of Psych uses the `permitted_classes:` parameter
39
- if YAML.method(:safe_load).parameters.map(&:last).include? :permitted_classes
40
- YAML.safe_load(yaml, permitted_classes: classes)
41
- else
42
- YAML.safe_load(yaml, classes)
43
- end
44
- end
45
-
46
31
  extend self
47
32
  end
data/relaton_bipm.gemspec CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
12
12
  "bibliographic use using the BibliographicItem model"
13
13
  spec.homepage = "https://github.com/relaton/relaton-bipm"
14
14
  spec.license = "BSD-2-Clause"
15
- spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
15
+ spec.required_ruby_version = Gem::Requirement.new(">= 2.6.0")
16
16
 
17
17
  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
18
18
 
@@ -33,18 +33,16 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
33
33
  spec.require_paths = ["lib"]
34
34
 
35
35
  spec.add_development_dependency "byebug"
36
- # spec.add_development_dependency "debase"
37
36
  spec.add_development_dependency "equivalent-xml", "~> 0.6"
38
37
  spec.add_development_dependency "pry-byebug"
39
- # spec.add_development_dependency "rake", "~> 13.0"
40
- # spec.add_development_dependency "ruby-debug-ide"
41
38
  spec.add_development_dependency "ruby-jing"
42
39
  spec.add_development_dependency "simplecov"
43
40
  spec.add_development_dependency "vcr"
44
41
  spec.add_development_dependency "webmock"
45
42
 
43
+ spec.add_dependency "faraday", "~> 1.0"
46
44
  spec.add_dependency "mechanize", "~> 2.8.0"
47
- spec.add_dependency "relaton-bib", "~> 1.11.0"
45
+ spec.add_dependency "relaton-bib", "~> 1.12.0"
48
46
  spec.add_dependency "rubyzip", "~> 2.3.0"
49
47
  spec.add_dependency "serrano", "~> 1.0"
50
48
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: relaton-bipm
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.11.9
4
+ version: 1.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ribose Inc.
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-06-09 00:00:00.000000000 Z
11
+ date: 2022-06-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: byebug
@@ -108,6 +108,20 @@ dependencies:
108
108
  - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: faraday
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '1.0'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '1.0'
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: mechanize
113
127
  requirement: !ruby/object:Gem::Requirement
@@ -128,14 +142,14 @@ dependencies:
128
142
  requirements:
129
143
  - - "~>"
130
144
  - !ruby/object:Gem::Version
131
- version: 1.11.0
145
+ version: 1.12.0
132
146
  type: :runtime
133
147
  prerelease: false
134
148
  version_requirements: !ruby/object:Gem::Requirement
135
149
  requirements:
136
150
  - - "~>"
137
151
  - !ruby/object:Gem::Version
138
- version: 1.11.0
152
+ version: 1.12.0
139
153
  - !ruby/object:Gem::Dependency
140
154
  name: rubyzip
141
155
  requirement: !ruby/object:Gem::Requirement
@@ -220,7 +234,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
220
234
  requirements:
221
235
  - - ">="
222
236
  - !ruby/object:Gem::Version
223
- version: 2.5.0
237
+ version: 2.6.0
224
238
  required_rubygems_version: !ruby/object:Gem::Requirement
225
239
  requirements:
226
240
  - - ">="