bipm-data-importer 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3bf2e9209301b13261c1b15013611cd17cf5355d6152c1f59d131aa486bbec6c
-  data.tar.gz: d1bf3ab0059a945f134068cf3c9cab045fdb6c9970edd3dc0cb2ce839c3dce9b
+  metadata.gz: 738de7ebf04b508b4a769577c1c7f82eb81e48f1c401e0e7895197cbb33b65c0
+  data.tar.gz: 37d6c10ed703778e8dd4c968ac2a3e174cc6cc937f6ab0bffb1ede6a3448d0c2
 SHA512:
-  metadata.gz: f653f802d49a88b5e9f0f5b6380a1c9a0fdd73e52be093cae3bc62020d9057221dc9b9264a75bf10198b154f16dd8010480056ae624f7a1c49e7659e1b87b6fc
-  data.tar.gz: dd797598112540212aa451e62089b4dde8dc89e4cbfd85e83e6c653e002e65fbab8783ca4cfaab6d8c10a4442e90cacbb6c057b02615c270bf6b7dc3f531a3e0
+  metadata.gz: '06043389fc63d5038a836342accce06736f9fcb68e61aa309ce980ae439f69d66dd79ffec35003c2d6ea121a5d727bb06cb453a4860d1760adedaf52db9c98f7'
+  data.tar.gz: 35e740c67821496b97012c251291bf755b19be2983f43f12129cc94b7ef3494fe8d51858269942ec275cb9e28bdec3f6f4d92049dc0c3bfd06e44ee6ed91f8e5

data/exe/bipm-fetch

@@ -29,22 +29,6 @@ bodies.each do |bodyid, bodyurl|
 
   body = bodyid.to_s.downcase.gsub(" ", "-").to_sym
 
-  meetings_en = VCR.use_cassette "#{body}/#{body}-meetings" do
-    a.get "#{bodyurl}/meetings"
-  end
-  
-  meetings_fr = VCR.use_cassette "#{body}/#{body}-meetings-fr" do
-    a.get "#{bodyurl.gsub("/en/", "/fr/")}/meetings"
-  end
-
-  publications_en = VCR.use_cassette "#{body}/#{body}-publications" do
-    a.get "#{bodyurl}/publications"
-  end
-
-  publications_fr = VCR.use_cassette "#{body}/#{body}-publications-fr" do
-    a.get "#{bodyurl.gsub("/en/", "/fr/")}/publications"
-  end
-
   resolutions = {}
   %w[en fr].each do |meeting_lang|
     next if ARGV[0] == '--fork' && fork
@@ -52,8 +36,45 @@ bodies.each do |bodyid, bodyurl|
     meeting_lang_sfx     = (meeting_lang == 'fr') ? "-fr" : ""
     meeting_lang_sfx_dir = (meeting_lang == 'fr') ? "-fr" : "-en"
 
-    meetings = (meeting_lang == 'en') ? meetings_en : meetings_fr
-    publications = (meeting_lang == 'en') ? publications_en : publications_fr
+    bodyurl_local = meeting_lang == "en" ? bodyurl : bodyurl.gsub("/en/", "/fr/")
+
+    cassfx = meeting_lang == "en" ? "" : "-fr"
+
+    pages = {}
+
+    pages[:index] = VCR.use_cassette "#{body}/#{body}-index#{meeting_lang_sfx}" do
+      a.get "#{bodyurl_local}"
+    end
+
+    pages[:meetings] = VCR.use_cassette "#{body}/#{body}-meetings#{meeting_lang_sfx}" do
+      a.get "#{bodyurl_local}/meetings"
+    end
+
+    pages[:publications] = VCR.use_cassette "#{body}/#{body}-publications#{meeting_lang_sfx}" do
+      a.get "#{bodyurl_local}/publications"
+    end
+
+    # CIPM
+    pages[:recommendations] = VCR.use_cassette "#{body}/#{body}-recommendations#{meeting_lang_sfx}" do
+      a.get "#{bodyurl_local}/recommendations"
+    rescue Mechanize::ResponseCodeError
+      nil
+    end
+
+    # CIPM has outcomes, JCRB has meeting-outcomes
+    # As of 2024-12, no other body has this special case.
+    outcomes_path = bodyid == :CIPM ? "outcomes" : "meeting-outcomes"
+
+    pages[:outcomes] = VCR.use_cassette "#{body}/#{body}-outcomes#{meeting_lang_sfx}" do
+      a.get "#{bodyurl_local}/#{outcomes_path}"
+    rescue Mechanize::ResponseCodeError
+      nil
+    end
+
+    meetings = pages[:meetings]
+    publications = pages[:publications]
+    recommendations = pages[:recommendations]
+    outcomes = pages[:outcomes]
 
     index = {
               "meetings" => {"fr" => [], "en" => []}, 
@@ -116,6 +137,17 @@ bodies.each do |bodyid, bodyurl|
           res_div.at_css('a').attr('href')
         end
 
+        resolutions_additional = recommendations&.css(".bipm-resolutions .publications__content")&.map do |res_div|
+          href = res_div.at_css('a').attr('href')
+
+          # bad case of french data...
+          href = href.gsub('/106-2017/', '/104-_1-2015/') if href =~ %r"/ci/cipm/106-2017/resolution-[12]\z"
+
+          href
+        end&.select do |href|
+          href.include? "/#{ident}/"
+        end || []
+
         # A mistake on a website, resolution 2 listed twice...
         # https://www.bipm.org/fr/committees/ci/cipm/94-2005/
         if [bodyid, meeting_lang, meeting_id] == [:CIPM, 'fr', '94'] && resolutions.sort.uniq != resolutions.sort
@@ -124,6 +156,8 @@ bodies.each do |bodyid, bodyurl|
           end
         end
 
+        resolutions = (resolutions + resolutions_additional).uniq
+
         h["resolutions"] = resolutions.map do |href|
           href = href.gsub('/web/guest/', "/#{meeting_lang}/")
           href = href.sub("www.bipm.org/", "www.bipm.org/#{meeting_lang}/") unless href.include? "/#{meeting_lang}/"
@@ -160,7 +194,31 @@ bodies.each do |bodyid, bodyurl|
 
         h["metadata"]["workgroup"] = wg if wg
 
-        h["resolutions"] = meeting.css('.bipm-decisions .decisions').map do |titletr|
+        decisions = meeting.css('.bipm-decisions .decisions')
+
+        # For some bodies, decisions/outcomes are on a different page altogether.
+        # But then we must select only decisions pertaining to our meeting.
+        if outcomes
+          decisions_additional = outcomes.css('.bipm-decisions .decisions')
+
+          decisions_additional = decisions_additional.select do |i|
+            pass = true if i["data-meeting_key"] == meeting_id
+            pass = true if i["data-meeting_key"] == "#{meeting_id}-0" # Some are with a 0, some without
+            pass = true if i["data-meeting"] == meeting_id # Some don't have meeting_key set, but have meeting set
+
+            pass
+          end
+
+          decisions = decisions.to_a + decisions_additional.to_a
+
+          # duplicates check...
+          duplicates = decisions.map{|i|i.at_css('.title-third').text}
+          if duplicates != duplicates.uniq
+            pp [:duplicates_found, decisions]
+          end
+        end
+
+        h["resolutions"] = decisions.map do |titletr|
           title = titletr.at_css('.title-third').text.strip
 
           type = case title
@@ -176,6 +234,9 @@ bodies.each do |bodyid, bodyurl|
             "decision"
           end
 
+          categories = titletr.attr('data-decisioncategories')
+          categories ||= "[]"
+
           r = {
             "dates" => [date.to_s],
             "subject" => bodyid.to_s,
@@ -185,7 +246,7 @@ bodies.each do |bodyid, bodyurl|
             "url" => meeting.uri.to_s,
             #TODO: "reference" => meeting.uri.merge(titletr.attr('data-link')).to_s,
 
-            "categories" => JSON.parse(titletr.attr('data-decisioncategories')).map(&:strip).uniq,
+            "categories" => JSON.parse(categories).map(&:strip).uniq,
 
             "considerations" => [],
             "actions" => [],
@@ -329,6 +390,9 @@ bodies.each do |bodyid, bodyurl|
       wg = hs.first["metadata"]["workgroup"]
       if wg
         fn = "#{BASE_DIR}/#{body}/workgroups/#{wg}/meetings#{meeting_lang_sfx_dir}/meeting-#{mid}.yml"
+      elsif body == :cgpm
+        # CGPM old script used numbering like 00, 01, ..., 11, ...
+        fn = "#{BASE_DIR}/#{body}/meetings#{meeting_lang_sfx_dir}/meeting-#{"%02d" % mid}.yml"
       else
         fn = "#{BASE_DIR}/#{body}/meetings#{meeting_lang_sfx_dir}/meeting-#{mid}.yml"
       end

data/lib/bipm/data/importer/asciimath.rb

@@ -9,10 +9,10 @@ module Bipm
         SPACE_AFTER=/(?:\Z|(?= |\s|[,()\/.~"^]))/
 
         PREFIXES = /m|c|d|k|M|G|T|/
-        UNITS = /t|m|mol|cal|µ|s|g|W|cd|Hz|J|K|N|V|H|A|C|F|T|Wb|sr|lx|lm|bar|sb|h|rad|°C|°F|°K/
+        UNITS = /t|m|mol|cal|µ|s|g|W|cd|Hz|J|K|N|V|H|A|C|F|T|Wb|sr|lx|lm|bar|sb|h|rad|fm|°C|°F|°K/
 
         def asciidoc_extract_math str
-          str.gsub(/\b_(#{MATH}{1,3})_/, 'stem:[\1]')
+          str.gsub(/\b_?_(#{MATH}{1,3})_?_/, 'stem:[\1]')
             .gsub("_,_", ',') # Some mistake in formatting
             .gsub("^er^", 'ESCUPerESCUP') # French specialities
             .gsub(/(bar|A) (table|of|key|de|being|full|1)( |,)/, 'ESC\1 \2\3') # A is Ampere, but also a particle, bar is a bar but also a bar
@@ -69,4 +69,4 @@ module Bipm
       end
     end
   end
-end
+end

data/lib/bipm/data/importer/common.rb

@@ -338,7 +338,7 @@ module Bipm
 
                 listmarker = nil
                 listitems = []
-                if (i["message"].split(/(?<!\+)\n/).all? { |j|
+                if (i["message"].split(/(?<!\+)\n(?!\+)/).all? { |j|
                   case j
                   when /\A\s*[*_]?#{PREFIX}#{kk}/i
                     true

data/lib/bipm/data/importer/version.rb

@@ -3,7 +3,7 @@
 module Bipm
   module Data
     module Importer
-      VERSION = "0.2.1"
+      VERSION = "0.2.2"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bipm-data-importer
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Ribose
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-12-08 00:00:00.000000000 Z
+date: 2024-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri