bipm-data-importer 0.1.3 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fcde8a899380a032040ac0f44f380edda5b0f0a6c6591dc874a9e4f90504544
-  data.tar.gz: ba4cce57619f4356eb66309d75347fa24db76ba67936fb0e0cc161b97745c13e
+  metadata.gz: 3bf2e9209301b13261c1b15013611cd17cf5355d6152c1f59d131aa486bbec6c
+  data.tar.gz: d1bf3ab0059a945f134068cf3c9cab045fdb6c9970edd3dc0cb2ce839c3dce9b
 SHA512:
-  metadata.gz: db266f37ca351605df8e13ea5c75fae2fe9b9ddbd8038e4392233e2e103d28e1a33c11a7f1bf397c93a8e5f32afdb293045341a776fd09f16b4cae5a054ac340
-  data.tar.gz: 46bec77135211aec51a90a03f21dbc72f9cc1443108b965be211449f89b963d90f87a58e354a5a33ec9a750fa0c0ea6357382f4ecbf1f76434c54bd3b3818a0f
+  metadata.gz: f653f802d49a88b5e9f0f5b6380a1c9a0fdd73e52be093cae3bc62020d9057221dc9b9264a75bf10198b154f16dd8010480056ae624f7a1c49e7659e1b87b6fc
+  data.tar.gz: dd797598112540212aa451e62089b4dde8dc89e4cbfd85e83e6c653e002e65fbab8783ca4cfaab6d8c10a4442e90cacbb6c057b02615c270bf6b7dc3f531a3e0

data/bipm-data-importer.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "nokogiri"
   spec.add_dependency "mechanize"
-  spec.add_dependency "reverse_adoc"
+  spec.add_dependency "coradoc"
   spec.add_dependency "pry"
 
   spec.add_dependency "vcr"

data/exe/bipm-fetch

@@ -15,7 +15,8 @@ bodies = {
   "CCL":       'https://www.bipm.org/en/committees/cc/ccl',
   "CCEM":      'https://www.bipm.org/en/committees/cc/ccem',
   "CCAUV":     'https://www.bipm.org/en/committees/cc/ccauv',
-  "CIPM":      'https://www.bipm.org/en/committees/ci/cipm'
+  "CIPM":      'https://www.bipm.org/en/committees/ci/cipm',
+  "CGPM":      'https://www.bipm.org/en/committees/cg/cgpm',
 }
 
 BASE_DIR = "data"
@@ -64,6 +65,7 @@ bodies.each do |bodyid, bodyurl|
 
       title = meeting_div.at_css('.meetings-list__informations-title').text.strip
       href = meeting_div.at_css('.meetings-list__informations-title').attr('href')
+      href = "/#{meeting_lang}" + href unless href.start_with? "/#{meeting_lang}/"
 
       ident = href.split("/#{body}/").last.gsub('/', '.')
       yr = href.include?("/wg/") ? nil : href.split('-').last
@@ -124,6 +126,7 @@ bodies.each do |bodyid, bodyurl|
 
         h["resolutions"] = resolutions.map do |href|
           href = href.gsub('/web/guest/', "/#{meeting_lang}/")
+          href = href.sub("www.bipm.org/", "www.bipm.org/#{meeting_lang}/") unless href.include? "/#{meeting_lang}/"
 
           # error: https://www.bipm.org/fr/committees/ci/cipm/104-_1-2015 has wrong references to Recommandations
           href = href.gsub('/104-2015/', '/104-_1-2015/')
@@ -343,7 +346,7 @@ bodies.each do |bodyid, bodyurl|
         "documents" => i.css(".publications__content").map do |d|
           {
             "title" => d.at_css(".title-third").text.strip.gsub(/\s+/, ' '),
-            "pdf" => d.at_css(".title-third").attr("href").split('?').first,
+            "pdf" => d.at_css(".title-third")&.attr("href")&.split('?')&.first,
             # "description" => d.css('.publications__body')[0]&.text&.strip,
             # "author" => d.css('.publications__body')[1]&.text&.strip,
           }.compact

data/exe/bipm-fetch-cgpm

@@ -1,71 +1,3 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/bipm-data-importer'
-
-BASE_DIR = "data"
-a = Mechanize.new
-
-meetings_en = VCR.use_cassette 'cgpm-meetings' do
-  a.get "https://www.bipm.org/en/committees/cg/cgpm/cgpm-resolutions"
-end
-
-meetings_fr = VCR.use_cassette 'cgpm-meetings-fr' do
-  a.get "https://www.bipm.org/fr/committees/cg/cgpm/cgpm-resolutions"
-end
-
-FileUtils.rm_rf "#{BASE_DIR}/cgpm/meetings"
-FileUtils.rm_rf "#{BASE_DIR}/cgpm/meetings-fr"
-FileUtils.rm_rf "#{BASE_DIR}/cgpm/meetings-en"
-
-[['en', meetings_en], ['fr', meetings_fr]].each do |meeting_lang, meetings|
-  urls = meetings.css('div.publications__content').map do |option|
-    url = option.at_css('a').attr('href')
-    url = url.gsub('/web/guest/', "/#{meeting_lang}/")
-    url.split('/').first(8).join('/')
-  end.uniq
-
-  urls.each do |url|
-    meeting_id = url.split('/').last.to_i
-    meeting_lang_sfx     = (meeting_lang == 'fr') ? "-fr" : ""
-    meeting_lang_sfx_dir = (meeting_lang == 'fr') ? "-fr" : "-en"
-    meeting = VCR.use_cassette("cgpm-meeting-#{meeting_id}#{meeting_lang_sfx}") { a.get url }
-
-    title = meeting.at_css('h1.session__title, .journal-content-article h1').text.strip
-    date = Bipm::Data::Importer::Common.extract_date(meeting.at_css('p.session__date, .journal-content-article h2').text)
-
-    pdf = Bipm::Data::Importer::Common.extract_pdf(meeting, meeting_lang)
-
-    h = {
-      "metadata" => {
-        "title" => title,
-        "identifier" => meeting_id,
-        "date" => date.to_s,
-        "source" => "BIPM - Pavillon de Breteuil",
-        "url" => meeting.uri.to_s
-      }
-    }
-
-    h["pdf"] = pdf if pdf
-
-    resolutions = meeting.links_with(href: %r</resolution->).map(&:href)
-
-    # A mistake on a website, resolution 5 listed 4 times...
-    # https://www.bipm.org/fr/committees/cg/cgpm/8-1933
-    if meeting_id == 8 && meeting_lang == 'fr' && resolutions.sort.uniq != resolutions.sort
-      resolutions = (1..15).map do |i|
-        "https://www.bipm.org/en/committees/cg/cgpm/8-1933/resolution-#{i}"
-      end
-    end
-
-    h["resolutions"] = resolutions.map do |res_link|
-      res_id = (res_link.split('-')[2] || 0).to_i
-      res_link = res_link.gsub('/web/guest/', "/#{meeting_lang}/")
-      res = VCR.use_cassette("cgpm-resolution-#{meeting_id}-#{res_id}#{meeting_lang_sfx}") { a.get res_link }
-
-      Bipm::Data::Importer::Common.parse_resolution(res, res_id, date, :cgpm, meeting_lang, "resolution?")
-    end
-
-    FileUtils.mkdir_p("#{BASE_DIR}/cgpm/meetings#{meeting_lang_sfx_dir}")
-    File.write("#{BASE_DIR}/cgpm/meetings#{meeting_lang_sfx_dir}/meeting-#{"%02d" % meeting_id}.yml", YAML.dump(h))
-  end
-end
+warn "DEPRECATED: This tool is no longer needed. Please call `bipm-fetch` only."

data/lib/bipm/data/importer/common.rb

@@ -1,20 +1,19 @@
-require 'mechanize'
-require 'reverse_adoc'
-require 'vcr'
-require 'date'
-require 'fileutils'
-require 'pry'
-require_relative 'asciimath'
+require "mechanize"
+require "coradoc/input/html"
+require "vcr"
+require "date"
+require "fileutils"
+require "pry"
+require_relative "asciimath"
 
 VCR.configure do |c|
-  c.cassette_library_dir = __dir__+'/../../../../cassettes'
+  c.cassette_library_dir = __dir__ + "/../../../../cassettes"
   c.hook_into :webmock
 end
 
 module Bipm
   module Data
     module Importer
-
       CONSIDERATIONS = {
         /(?:having(?: regard)?|ayant|concerne|vu la|agissant conformément|sachant|de porter)/i => "having / having regard",
         /(?:noting|to note|took note|note[sd]?|taking note|takes note|constatant|constate|that|notant|notant que|note également|(?:prend|prenant) (?:acte|note))/i => "noting",
@@ -50,7 +49,7 @@ module Bipm
         /(?:Appendix 1 of the|L'Annexe 1 de la|increased|a (?:examiné|préparé)|transmettra|fournira|increased|developed a document|prendra contact)/i => "declares",
         /(?:Le Temps Atomique International |International Atomic Time \(TAI\) |will meet )/i => "declares",
         /(?:ask[s ]|asked|souhaite|souhaiterait)/i => "asks",
-        /(?:(?:further )?invit(?:[ée][ds]?|era)|renouvelle en conséquence|convient d'inviter)/i => "invites / further invites",
+        /(?:(?:further |et )?invit(?:[ée][ds]?|era)|renouvelle en conséquence|convient d'inviter)/i => "invites / further invites",
         /(?:resolve[sd]?)/i => "resolves",
         /(?:confirms|confirmed?|confirme que|committed|s'engageant)/i => "confirms",
         /(?:welcom(?:e[sd]?|ing)|accueille favorablement(?:les)?|salu(?:e|ant))/i => "welcomes",
@@ -79,50 +78,52 @@ module Bipm
         /(?:empowers|habilite)/i => "empowers",
       }
 
-      PREFIX1=/(?:The|Le) CIPM |La Conférence |M. Volterra |M. le Président |unanimously |would |a |sont |will |were |did not |strongly |(?:La|The) (?:\d+(?:e|th)|Quinzième) Conférence Générale des Poids et Mesures(?: a |,\s+)?/i
-      PREFIX2=/The \d+th Conférence Générale des Poids et Mesures |The Conference |and |et (?:en |)|has |renouvelle sa |renews its |further |and further |En ce qui |après avoir |\.\.\.\n+\t*/i
-      PREFIX3=/Sur la proposition de M. le Président, la convocation de cette Conférence de Thermométrie est |Le texte corrigé, finalement |(?:The|Le) Comité International(?: des Poids et Mesures)?(?: \(CIPM\))?(?: a |,)?\s*/i
-      PREFIX4=/(?:The |Le |)(?:JCRB|JCGM|CCU|CCTF|CCT|CCRI|CCPR|CCQM|CCM|CCL|CCEM|CCAUV|KCDB),? (?:also |)|Each RMO |fully |The JCRB Rules of Procedure are |Bob Watters and Claudine Thomas /
-      PREFIX5=/(?:The |Le |All |)(?:incoming |)(?:JCRB |KCDB |)(?:documents|(?:Consultative |)Committees?|Office|Chairman(?: and Secretary|)|Joint BIPM[\/-]ILAC Working Group(?: \(see Action 22\))|RMO(?:[- ]JCRB|) Representatives(?: to the JRCB|)|(?:BIPM |)Director(?: of BIPM|)|SIM|(?:Exec(?:utive|) |)Secretary(?:\(ies\)|)|RMOs, except SIM,|RMOs|APMP|\(?(?:[MD]r|Prof) [A-Z][a-zR-]+\)?|CMCs|EUR[AO]MET|COOMET|GULFMET) |It was /
-      PREFIX6=/“|"|« à |All RMO documents related to review procedures |Mr Lam and Dr Kühne |The Prof. Kühne, Mr Jones and the Executive Secretary |Ajchara Charoensook, from APMP, /
+      PREFIX1 = /(?:The|Le) CIPM |La Conférence |M. Volterra |M. le Président |unanimously |would |a |sont |will |were |did not |strongly |(?:La|The) (?:\d+(?:e|th)|Quinzième) Conférence Générale des Poids et Mesures(?: a |,\s+)?/i
+      PREFIX2 = /The \d+th Conférence Générale des Poids et Mesures |The Conference |and |et (?:en |)|has |renouvelle sa |renews its |further |and further |En ce qui |après avoir |\.\.\.\n+\t*/i
+      PREFIX3 = /Sur la proposition de M. le Président, la convocation de cette Conférence de Thermométrie est |Le texte corrigé, finalement |(?:The|Le) Comité International(?: des Poids et Mesures)?(?: \(CIPM\))?(?: a |,)?\s*/i
+      PREFIX4 = /(?:The |Le |)(?:JCRB|JCGM|CCU|CCTF|CCT|CCRI|CCPR|CCQM|CCM|CCL|CCEM|CCAUV|KCDB),? (?:also |)|Each RMO |fully |The JCRB Rules of Procedure are |Bob Watters and Claudine Thomas /
+      PREFIX5 = /(?:The |Le |All |)(?:incoming |)(?:JCRB |KCDB |)(?:documents|(?:Consultative |)Committees?|Office|Chairman(?: and Secretary|)|Joint BIPM[\/-]ILAC Working Group(?: \(see Action 22\))|RMO(?:[- ]JCRB|) Representatives(?: to the JRCB|)|(?:BIPM |)Director(?: of BIPM|)|SIM|(?:Exec(?:utive|) |)Secretary(?:\(ies\)|)|RMOs, except SIM,|RMOs|APMP|\(?(?:[MD]r|Prof) [A-Z][a-zR-]+\)?|CMCs|EUR[AO]MET|COOMET|GULFMET) |It was /
+      PREFIX6 = /“|"|« à |All RMO documents related to review procedures |Mr Lam and Dr Kühne |The Prof. Kühne, Mr Jones and the Executive Secretary |Ajchara Charoensook, from APMP, /
+
+      PREFIX = /(?:#{PREFIX1}|#{PREFIX2}|#{PREFIX3}|#{PREFIX4}|#{PREFIX5}|#{PREFIX6})?/i
 
-      PREFIX=/(?:#{PREFIX1}|#{PREFIX2}|#{PREFIX3}|#{PREFIX4}|#{PREFIX5}|#{PREFIX6})?/i
+      SUFFIX = / (?:that|que)\b|(?: (?:the |that |le |que les )?((?:[A-Z]|national|laboratoires).{0,80}?)(?: to)?\b|)/
 
-      SUFFIX=/ (?:that|que)\b|(?: (?:the |that |le |que les )?((?:[A-Z]|national|laboratoires).{0,80}?)(?: to)?\b|)/
+      DOIREGEX = %r'\s+<p>\s+<b>DOI :</b> (.*?)\s+</p>\n\n'
 
       module Common
-        def replace_links ps, res, lang
-          ps.css('a[href]').each do |a|
-            href = a.attr('href')
+        def replace_links(ps, res, lang)
+          ps.css("a[href]").each do |a|
+            href = a.attr("href")
 
-            href = href.gsub(%r'\Ahttps://www.bipm.org/', '')
+            href = href.gsub(%r'\Ahttps://www\.bipm\.org/', "")
 
             # Correct links
-            href = href.gsub('/web/guest/', "/#{lang}/")
+            href = href.gsub("/web/guest/", "/#{lang}/")
 
             # Account for some mistakes from an upstream document
-            href = href.gsub(%r"\A/jen/", '/en/')
-            href = href.gsub(%r"\A/en/CGPM/jsp/", '/en/CGPM/db/')
+            href = href.gsub(%r"\A/jen/", "/en/")
+            href = href.gsub(%r"\A/en/CGPM/jsp/", "/en/CGPM/db/")
 
             href = case href
-            when %r'\A/(\w{2})/CGPM/db/(\d+)/(\d+)/(#.*)?\z',
-                 %r'\A/jsp/(\w{2})/ViewCGPMResolution\.jsp\?CGPM=(\d+)&RES=(\d+)(#.*)?\z',
-                 %r'\A/(\w{2})/committees/cg/cgpm/(\d+)-\d+/resolution-(\d+)(#.*)?\z',
-              "cgpm-resolution:#{$1}/#{$2}/#{$3}#{$4}"
-            when %r'\A/(\w{2})/CIPM/db/(\d+)/(\d+)/(#.*)?\z'
-              "cipm-resolution:#{$1}/#{$2}/#{$3}#{$4}"
-            when %r'\A/(\w{2})/committees/cipm/meeting/([0-9()I]+).html(#.*)?\z'
-              "cipm-decisions:#{$1}/#{$2}#{$3}"
-            else
-              URI(res.uri).merge(href).to_s # Relative -> absolute
-            end
+              when %r'\A/(\w{2})/CGPM/db/(\d+)/(\d+)/(#.*)?\z',
+                   %r'\A/jsp/(\w{2})/ViewCGPMResolution\.jsp\?CGPM=(\d+)&RES=(\d+)(#.*)?\z',
+                   %r'\A/(\w{2})/committees/cg/cgpm/(\d+)-\d+/resolution-(\d+)(#.*)?\z',
+                   "cgpm-resolution:#{$1}/#{$2}/#{$3}#{$4}"
+              when %r'\A/(\w{2})/CIPM/db/(\d+)/(\d+)/(#.*)?\z'
+                "cipm-resolution:#{$1}/#{$2}/#{$3}#{$4}"
+              when %r'\A/(\w{2})/committees/cipm/meeting/([0-9()I]+).html(#.*)?\z'
+                "cipm-decisions:#{$1}/#{$2}#{$3}"
+              else
+                URI(res.uri).merge(href).to_s # Relative -> absolute
+              end
 
-            a.set_attribute('href', href)
+            a.set_attribute("href", href)
           end
         end
 
-        def replace_centers ps
-          centers = ps.css('center').to_a
+        def replace_centers(ps)
+          centers = ps.css("center").to_a
           while centers.length > 0
             center = centers.first
             current = center
@@ -131,7 +132,7 @@ module Bipm
               break unless current.next
               while Nokogiri::XML::Text === current.next
                 current = current.next
-                break if current.text.strip != ''
+                break if current.text.strip != ""
               end
               break unless current.next
               break unless current.next.name == "center"
@@ -157,47 +158,52 @@ module Bipm
           end
 
           # Remove the remaining centers
-          ps.css('center').each do |i|
+          ps.css("center").each do |i|
             i.replace i.inner_html
           end
         end
 
-        def format_message part
+        def format_message(part)
           AsciiMath.asciidoc_extract_math(
-            ReverseAdoc.convert(part).strip.gsub("&nbsp;", ' ').gsub(" \n", "\n")
+            Coradoc::Input::HTML.convert(part).strip.gsub("&nbsp;", " ").gsub(" \n", "\n")
           )
+        rescue
+          warn "Bug in Coradoc, couldn't parse the following document:"
+          pp part
+          warn "Please report this as an issue to https://github.com/metanorma/coradoc"
+          raise
         end
 
-        def ng_to_string ps
-          ps.inner_html.encode('utf-8').gsub("\r", '').gsub(%r'</?nobr>','')
+        def ng_to_string(ps)
+          ps.inner_html.encode("utf-8").gsub("\r", "").gsub(%r'</?nobr>', "")
         end
 
-        def parse_resolution res, res_id, date, type = :cgpm, lang = 'en', rec_type = nil
+        def parse_resolution(res, res_id, date, type = :cgpm, lang = "en", rec_type = nil)
           # Reparse the document after fixing upstream syntax
           fixed_body = res.body.gsub("<name=", "<a name=")
-          fixed_body = fixed_body.force_encoding('utf-8')
-          fixed_body = fixed_body.gsub('&Eacute;', 'É')
-          fixed_body = fixed_body.gsub('&#171;&#032;', '« ')
-          fixed_body = fixed_body.gsub('&#032;&#187;', ' »')
-          fixed_body = fixed_body.sub(%r'<h1>(.*?)</h1>'m, '')
+          fixed_body = fixed_body.force_encoding("utf-8")
+          fixed_body = fixed_body.gsub("&Eacute;", "É")
+          fixed_body = fixed_body.gsub("&#171;&#032;", "« ")
+          fixed_body = fixed_body.gsub("&#032;&#187;", " »")
+          fixed_body = fixed_body.sub(%r'<h1>(.*?)</h1>'m, "")
           supertitle = $1.strip
-          fixed_body = fixed_body.sub(%r'<h2>(.*?)</h2>'m, '')
+          fixed_body = fixed_body.sub(%r'<h2>(.*?)</h2>'m, "")
           title = $1.strip
           fixed_body = fixed_body.sub(/(="web-content">)\s*<p>\s*(<p)/, '\1\2')
           fixed_body = fixed_body.gsub(%r"<a name=\"haut\">(.*?)</a>"m, '\1')
           ng = Nokogiri::HTML(fixed_body, res.uri.to_s, "utf-8", Nokogiri::XML::ParseOptions.new.default_html.noent)
 
-          refs = ng.css('.publication-card_reference a')
+          refs = ng.css(".publication-card_reference a")
 
           if rec_type.end_with? "?"
             rec_type = case supertitle
-                       when /\AD[eé]claration/
-                         "declaration"
-                       when /\AR[eé]solution/
-                         "resolution"
-                       else
-                         rec_type[..-2]
-                       end
+              when /\AD[eé]claration/
+                "statement"
+              when /\AR[eé]solution/
+                "resolution"
+              else
+                rec_type[..-2]
+              end
           end
 
           r = {
@@ -220,7 +226,7 @@ module Bipm
           r.delete("type") unless r["type"]
 
           if refs.length > 0
-            r["reference"] = res.uri.merge(refs.first.attr('href')).to_s.split('?').first
+            r["reference"] = res.uri.merge(refs.first.attr("href")).to_s.split("?").first
             name, page = refs.first.text.strip.split(/, p(?=[0-9])/)
             r["reference_name"] = name
             if page
@@ -234,7 +240,7 @@ module Bipm
             r.delete("reference_page")
           end
 
-          ps = ng.css('div.journal-content-article').first
+          ps = ng.css("div.journal-content-article").first
 
           #binding.pry if ps.count != 1
 
@@ -247,6 +253,11 @@ module Bipm
           doc = Common.ng_to_string(ps)
           # doc = AsciiMath.html_to_asciimath(doc)
 
+          if doc.match? DOIREGEX
+            doc = doc.sub(DOIREGEX, "")
+            r["doi"] = $1
+          end
+
           parts = doc.split(/(\n(?:<p>)?<b>.*?<\/b>|\n<p><i>.*?<\/i>|<div class="bipm-lame-grey">|<h3>|<p>(?:après examen |après avoir entendu )|having noted that |decides to define |décide de définir |conformément à l'invitation|acting in accordance with|recommande que les résultats|(?:strongly |and further |)(?:considers|recommends|recommande) (?:la|that|que(?! « ))|estime que|declares<\/p>|déclare :<\/b><\/p>|<a name="_ftn\d)/)
           nparts = [parts.shift]
           while parts.length > 0
@@ -267,13 +278,13 @@ module Bipm
               next
             end
 
-            if parse.start_with? 'NOTE'
-              part = part.sub('<h3>NOTE</h3>', '')
+            if parse.start_with? "NOTE"
+              part = part.sub("<h3>NOTE</h3>", "")
               r["notes"] = Common.format_message(part)
               next
             end
 
-            CONSIDERATIONS.any? do |k,v|
+            CONSIDERATIONS.any? do |k, v|
               if parse =~ /\A#{PREFIX}#{k}\b/i
                 r["considerations"] << prev = {
                   "type" => v,
@@ -283,7 +294,7 @@ module Bipm
               end
             end && next
 
-            ACTIONS.any? do |k,v|
+            ACTIONS.any? do |k, v|
               if parse =~ /\A#{PREFIX}#{k}\b/i
                 r["actions"] << prev = {
                   "type" => v,
@@ -314,13 +325,13 @@ module Bipm
           end
 
           %w[considerations actions].each do |type|
-            map = type == 'actions' ? ACTIONS : CONSIDERATIONS
+            map = type == "actions" ? ACTIONS : CONSIDERATIONS
             r[type] = r[type].map do |i|
               islist = false
 
               kk = nil
 
-              if map.any? { |k,v| (i["message"].split("\n").first =~ /\A\s*([*_]?)(#{PREFIX}#{k})\1?(#{SUFFIX})\1?\s*\z/i) && (kk = k) }
+              if map.any? { |k, v| (i["message"].split("\n").first =~ /\A\s*([*_]?)(#{PREFIX}#{k})\1?(#{SUFFIX})\1?\s*\z/i) && (kk = k) }
                 prefix = $2
                 suffix = $3
                 subject = $4
@@ -347,15 +358,15 @@ module Bipm
 
               if subject
                 #p subject
-                r['subject'] ||= []
-                r['subject'] << subject
+                r["subject"] ||= []
+                r["subject"] << subject
               end
 
               if islist
                 suffix = suffix.strip
-                suffix = nil if suffix == ''
+                suffix = nil if suffix == ""
                 listitems.map do |li|
-                  i.merge 'message' => [prefix, suffix, li].compact.join(" ")
+                  i.merge "message" => [prefix, suffix, li].compact.join(" ")
                 end
               else
                 i
@@ -363,13 +374,13 @@ module Bipm
             end.flatten
           end
 
-          if r['subject']
-            r['subject'] = r['subject'].uniq.join(" and ")
+          if r["subject"]
+            r["subject"] = r["subject"].uniq.join(" and ")
           end
 
           # Note: we replace the previously set r['subject'].
-          r['subject'] = type.to_s.upcase.gsub("-", ' ')
-          r['subject'] = 'CCDS' if type == :cctf && supertitle.include?("CCDS")
+          r["subject"] = type.to_s.upcase.gsub("-", " ")
+          r["subject"] = "CCDS" if type == :cctf && supertitle.include?("CCDS")
 
           r
         end
@@ -377,11 +388,11 @@ module Bipm
         def extract_pdf(meeting, lang)
           pdfs = meeting.css('a.title-third[href*=".pdf"]')
                         .map { |i| i.attr("href") }
-                        .map { |i| i.split('?').first }          
+                        .map { |i| i.split("?").first }
                         .select do |i|
-                          i.downcase.include?("-#{lang}.pdf") ||
-                          %w[en fr].none? { |l| i.downcase.include? "-#{l}.pdf" }
-                        end
+            i.downcase.include?("-#{lang}.pdf") ||
+            %w[en fr].none? { |l| i.downcase.include? "-#{l}.pdf" }
+          end
 
           pdfs = pdfs.first if pdfs.length <= 1
 
@@ -389,8 +400,10 @@ module Bipm
         end
 
         def extract_date(date_str)
+          return nil unless date_str
+
           date = date_str.strip
-                         .gsub(/\s+/, ' ')
+                         .gsub(/\s+/, " ")
                          .gsub("février", "february") # 3 first letters must match English
                          .gsub("juin", "june")
                          .gsub("avril", "april")
@@ -411,7 +424,6 @@ module Bipm
 
         extend self
       end
-
     end
   end
 end

data/lib/bipm/data/importer/version.rb

@@ -3,7 +3,7 @@
 module Bipm
   module Data
     module Importer
-      VERSION = "0.1.3"
+      VERSION = "0.2.1"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bipm-data-importer
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.1
 platform: ruby
 authors:
 - Ribose
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-01-06 00:00:00.000000000 Z
+date: 2024-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: reverse_adoc
+  name: coradoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -206,7 +206,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.3.27
 signing_key: 
 specification_version: 4
 summary: Importer for BIPM CGPM and CIPM content