bipm-data-importer 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fcde8a899380a032040ac0f44f380edda5b0f0a6c6591dc874a9e4f90504544
-  data.tar.gz: ba4cce57619f4356eb66309d75347fa24db76ba67936fb0e0cc161b97745c13e
+  metadata.gz: 3757c28f685035d3af9283b248cc57d523e9735789fe51753622d72962db84a0
+  data.tar.gz: 874b08f81363f1d27802e87550be8d73aae99cc5156d0b2f0df92b6b055ee780
 SHA512:
-  metadata.gz: db266f37ca351605df8e13ea5c75fae2fe9b9ddbd8038e4392233e2e103d28e1a33c11a7f1bf397c93a8e5f32afdb293045341a776fd09f16b4cae5a054ac340
-  data.tar.gz: 46bec77135211aec51a90a03f21dbc72f9cc1443108b965be211449f89b963d90f87a58e354a5a33ec9a750fa0c0ea6357382f4ecbf1f76434c54bd3b3818a0f
+  metadata.gz: c394b46052dc8d7dac1f15352176280573ef71c5cb5a72b11e8204499f1f7d7cfb23ac9b09edea208aa7ea3f48eb61f890affb518d3eaf6d79a30879b29fd9a5
+  data.tar.gz: f0eb33717e70a707e70c491c7c676a5e4b1769aec67b873bdfa2d57317d3c3bc17a7194c8ade7008a58aa8a296e038cddaad9d631490efdeb4bdf61655e9e2e4

data/bipm-data-importer.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "nokogiri"
   spec.add_dependency "mechanize"
-  spec.add_dependency "reverse_adoc"
+  spec.add_dependency "coradoc"
   spec.add_dependency "pry"
 
   spec.add_dependency "vcr"

data/exe/bipm-fetch

@@ -15,7 +15,8 @@ bodies = {
   "CCL":       'https://www.bipm.org/en/committees/cc/ccl',
   "CCEM":      'https://www.bipm.org/en/committees/cc/ccem',
   "CCAUV":     'https://www.bipm.org/en/committees/cc/ccauv',
-  "CIPM":      'https://www.bipm.org/en/committees/ci/cipm'
+  "CIPM":      'https://www.bipm.org/en/committees/ci/cipm',
+  "CGPM":      'https://www.bipm.org/en/committees/cg/cgpm',
 }
 
 BASE_DIR = "data"
@@ -64,6 +65,7 @@ bodies.each do |bodyid, bodyurl|
 
       title = meeting_div.at_css('.meetings-list__informations-title').text.strip
       href = meeting_div.at_css('.meetings-list__informations-title').attr('href')
+      href = "/#{meeting_lang}" + href unless href.start_with? "/#{meeting_lang}/"
 
       ident = href.split("/#{body}/").last.gsub('/', '.')
       yr = href.include?("/wg/") ? nil : href.split('-').last
@@ -124,6 +126,7 @@ bodies.each do |bodyid, bodyurl|
 
         h["resolutions"] = resolutions.map do |href|
           href = href.gsub('/web/guest/', "/#{meeting_lang}/")
+          href = href.sub("www.bipm.org/", "www.bipm.org/#{meeting_lang}/") unless href.include? "/#{meeting_lang}/"
 
           # error: https://www.bipm.org/fr/committees/ci/cipm/104-_1-2015 has wrong references to Recommandations
           href = href.gsub('/104-2015/', '/104-_1-2015/')
@@ -343,7 +346,7 @@ bodies.each do |bodyid, bodyurl|
         "documents" => i.css(".publications__content").map do |d|
           {
             "title" => d.at_css(".title-third").text.strip.gsub(/\s+/, ' '),
-            "pdf" => d.at_css(".title-third").attr("href").split('?').first,
+            "pdf" => d.at_css(".title-third")&.attr("href")&.split('?')&.first,
             # "description" => d.css('.publications__body')[0]&.text&.strip,
             # "author" => d.css('.publications__body')[1]&.text&.strip,
           }.compact

data/exe/bipm-fetch-cgpm

@@ -1,71 +1,3 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/bipm-data-importer'
-
-BASE_DIR = "data"
-a = Mechanize.new
-
-meetings_en = VCR.use_cassette 'cgpm-meetings' do
-  a.get "https://www.bipm.org/en/committees/cg/cgpm/cgpm-resolutions"
-end
-
-meetings_fr = VCR.use_cassette 'cgpm-meetings-fr' do
-  a.get "https://www.bipm.org/fr/committees/cg/cgpm/cgpm-resolutions"
-end
-
-FileUtils.rm_rf "#{BASE_DIR}/cgpm/meetings"
-FileUtils.rm_rf "#{BASE_DIR}/cgpm/meetings-fr"
-FileUtils.rm_rf "#{BASE_DIR}/cgpm/meetings-en"
-
-[['en', meetings_en], ['fr', meetings_fr]].each do |meeting_lang, meetings|
-  urls = meetings.css('div.publications__content').map do |option|
-    url = option.at_css('a').attr('href')
-    url = url.gsub('/web/guest/', "/#{meeting_lang}/")
-    url.split('/').first(8).join('/')
-  end.uniq
-
-  urls.each do |url|
-    meeting_id = url.split('/').last.to_i
-    meeting_lang_sfx     = (meeting_lang == 'fr') ? "-fr" : ""
-    meeting_lang_sfx_dir = (meeting_lang == 'fr') ? "-fr" : "-en"
-    meeting = VCR.use_cassette("cgpm-meeting-#{meeting_id}#{meeting_lang_sfx}") { a.get url }
-
-    title = meeting.at_css('h1.session__title, .journal-content-article h1').text.strip
-    date = Bipm::Data::Importer::Common.extract_date(meeting.at_css('p.session__date, .journal-content-article h2').text)
-
-    pdf = Bipm::Data::Importer::Common.extract_pdf(meeting, meeting_lang)
-
-    h = {
-      "metadata" => {
-        "title" => title,
-        "identifier" => meeting_id,
-        "date" => date.to_s,
-        "source" => "BIPM - Pavillon de Breteuil",
-        "url" => meeting.uri.to_s
-      }
-    }
-
-    h["pdf"] = pdf if pdf
-
-    resolutions = meeting.links_with(href: %r</resolution->).map(&:href)
-
-    # A mistake on a website, resolution 5 listed 4 times...
-    # https://www.bipm.org/fr/committees/cg/cgpm/8-1933
-    if meeting_id == 8 && meeting_lang == 'fr' && resolutions.sort.uniq != resolutions.sort
-      resolutions = (1..15).map do |i|
-        "https://www.bipm.org/en/committees/cg/cgpm/8-1933/resolution-#{i}"
-      end
-    end
-
-    h["resolutions"] = resolutions.map do |res_link|
-      res_id = (res_link.split('-')[2] || 0).to_i
-      res_link = res_link.gsub('/web/guest/', "/#{meeting_lang}/")
-      res = VCR.use_cassette("cgpm-resolution-#{meeting_id}-#{res_id}#{meeting_lang_sfx}") { a.get res_link }
-
-      Bipm::Data::Importer::Common.parse_resolution(res, res_id, date, :cgpm, meeting_lang, "resolution?")
-    end
-
-    FileUtils.mkdir_p("#{BASE_DIR}/cgpm/meetings#{meeting_lang_sfx_dir}")
-    File.write("#{BASE_DIR}/cgpm/meetings#{meeting_lang_sfx_dir}/meeting-#{"%02d" % meeting_id}.yml", YAML.dump(h))
-  end
-end
+warn "DEPRECATED: This tool is no longer needed. Please call `bipm-fetch` only."

data/lib/bipm/data/importer/common.rb

@@ -1,5 +1,5 @@
 require 'mechanize'
-require 'reverse_adoc'
+require 'coradoc/input/html'
 require 'vcr'
 require 'date'
 require 'fileutils'
@@ -50,7 +50,7 @@ module Bipm
         /(?:Appendix 1 of the|L'Annexe 1 de la|increased|a (?:examiné|préparé)|transmettra|fournira|increased|developed a document|prendra contact)/i => "declares",
         /(?:Le Temps Atomique International |International Atomic Time \(TAI\) |will meet )/i => "declares",
         /(?:ask[s ]|asked|souhaite|souhaiterait)/i => "asks",
-        /(?:(?:further )?invit(?:[ée][ds]?|era)|renouvelle en conséquence|convient d'inviter)/i => "invites / further invites",
+        /(?:(?:further |et )?invit(?:[ée][ds]?|era)|renouvelle en conséquence|convient d'inviter)/i => "invites / further invites",
         /(?:resolve[sd]?)/i => "resolves",
         /(?:confirms|confirmed?|confirme que|committed|s'engageant)/i => "confirms",
         /(?:welcom(?:e[sd]?|ing)|accueille favorablement(?:les)?|salu(?:e|ant))/i => "welcomes",
@@ -90,6 +90,8 @@ module Bipm
 
       SUFFIX=/ (?:that|que)\b|(?: (?:the |that |le |que les )?((?:[A-Z]|national|laboratoires).{0,80}?)(?: to)?\b|)/
 
+      DOIREGEX = %r'\s+<p>\s+<b>DOI :</b> (.*?)\s+</p>\n\n'
+
       module Common
         def replace_links ps, res, lang
           ps.css('a[href]').each do |a|
@@ -164,8 +166,13 @@ module Bipm
 
         def format_message part
           AsciiMath.asciidoc_extract_math(
-            ReverseAdoc.convert(part).strip.gsub("&nbsp;", ' ').gsub(" \n", "\n")
+            Coradoc::Input::HTML.convert(part).strip.gsub("&nbsp;", ' ').gsub(" \n", "\n")
           )
+        rescue
+          warn "Bug in Coradoc, couldn't parse the following document:"
+          pp part
+          warn "Please report this as an issue to https://github.com/metanorma/coradoc"
+          raise
         end
 
         def ng_to_string ps
@@ -247,6 +254,11 @@ module Bipm
           doc = Common.ng_to_string(ps)
           # doc = AsciiMath.html_to_asciimath(doc)
 
+          if doc.match? DOIREGEX
+            doc = doc.sub(DOIREGEX, '')
+            r["doi"] = $1
+          end
+
           parts = doc.split(/(\n(?:<p>)?<b>.*?<\/b>|\n<p><i>.*?<\/i>|<div class="bipm-lame-grey">|<h3>|<p>(?:après examen |après avoir entendu )|having noted that |decides to define |décide de définir |conformément à l'invitation|acting in accordance with|recommande que les résultats|(?:strongly |and further |)(?:considers|recommends|recommande) (?:la|that|que(?! « ))|estime que|declares<\/p>|déclare :<\/b><\/p>|<a name="_ftn\d)/)
           nparts = [parts.shift]
           while parts.length > 0
@@ -389,6 +401,8 @@ module Bipm
         end
 
         def extract_date(date_str)
+          return nil unless date_str
+
           date = date_str.strip
                          .gsub(/\s+/, ' ')
                          .gsub("février", "february") # 3 first letters must match English

data/lib/bipm/data/importer/version.rb

@@ -3,7 +3,7 @@
 module Bipm
   module Data
     module Importer
-      VERSION = "0.1.3"
+      VERSION = "0.2.0"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bipm-data-importer
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Ribose
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-01-06 00:00:00.000000000 Z
+date: 2024-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: reverse_adoc
+  name: coradoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -206,7 +206,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.3.27
 signing_key: 
 specification_version: 4
 summary: Importer for BIPM CGPM and CIPM content