bipm-data-importer 0.2.1 → 0.4.0

data/bipm-data-importer.gemspec

@@ -11,17 +11,17 @@ Gem::Specification.new do |spec|
   spec.summary = "Importer for BIPM CGPM and CIPM content"
   spec.description = "Importer for BIPM CGPM and CIPM content"
   spec.homepage = "https://github.com/metanorma/bipm-data-importer"
-  spec.required_ruby_version = ">= 2.5.0"
+  spec.required_ruby_version = ">= 3.3.0"
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/metanorma/bipm-data-importer"
   spec.metadata["changelog_uri"] = "https://github.com/metanorma/bipm-data-importer"
 
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject do |f|
-      (f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+      f == __FILE__ ||
+        f.match(%r{\A(?:spec|features|\.git|\.circleci|\.github)/}) ||
+        f.start_with?(".gitignore", ".rspec", ".rubocop")
     end
   end
   spec.bindir = "exe"
@@ -30,10 +30,14 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "nokogiri"
   spec.add_dependency "mechanize"
-  spec.add_dependency "coradoc"
-  spec.add_dependency "pry"
-
+  # coradoc 2.0 removed `coradoc/input/html` (called in
+  # lib/bipm/data/importer/common.rb) as part of a major refactor.
+  # Pinned to 1.x until a follow-up ports the HTML input calls to the
+  # coradoc 2.x API (or wherever that functionality has been extracted).
+  spec.add_dependency "coradoc", "~> 1.1"
   spec.add_dependency "vcr"
+
+  spec.add_development_dependency "pry"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"

data/exe/bipm-fetch

@@ -1,378 +1,4 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/bipm-data-importer'
-
-bodies = {
-  "JCRB":      'https://www.bipm.org/en/committees/jc/jcrb',
-  "JCGM":      'https://www.bipm.org/en/committees/jc/jcgm',
-  "CCU":       'https://www.bipm.org/en/committees/cc/ccu',
-  "CCTF":      'https://www.bipm.org/en/committees/cc/cctf',
-  "CCT":       'https://www.bipm.org/en/committees/cc/cct',
-  "CCRI":      'https://www.bipm.org/en/committees/cc/ccri',
-  "CCPR":      'https://www.bipm.org/en/committees/cc/ccpr',
-  "CCQM":      'https://www.bipm.org/en/committees/cc/ccqm',
-  "CCM":       'https://www.bipm.org/en/committees/cc/ccm',
-  "CCL":       'https://www.bipm.org/en/committees/cc/ccl',
-  "CCEM":      'https://www.bipm.org/en/committees/cc/ccem',
-  "CCAUV":     'https://www.bipm.org/en/committees/cc/ccauv',
-  "CIPM":      'https://www.bipm.org/en/committees/ci/cipm',
-  "CGPM":      'https://www.bipm.org/en/committees/cg/cgpm',
-}
-
-BASE_DIR = "data"
-a = Mechanize.new
-
-bodies.each do |bodyid, bodyurl|
-  next if ARGV[0] == '--fork' && fork
-
-  next if ARGV[0] && ARGV[0].start_with?("--body=") && ARGV[0].downcase != "--body=#{bodyid}".downcase
-
-  body = bodyid.to_s.downcase.gsub(" ", "-").to_sym
-
-  meetings_en = VCR.use_cassette "#{body}/#{body}-meetings" do
-    a.get "#{bodyurl}/meetings"
-  end
-  
-  meetings_fr = VCR.use_cassette "#{body}/#{body}-meetings-fr" do
-    a.get "#{bodyurl.gsub("/en/", "/fr/")}/meetings"
-  end
-
-  publications_en = VCR.use_cassette "#{body}/#{body}-publications" do
-    a.get "#{bodyurl}/publications"
-  end
-
-  publications_fr = VCR.use_cassette "#{body}/#{body}-publications-fr" do
-    a.get "#{bodyurl.gsub("/en/", "/fr/")}/publications"
-  end
-
-  resolutions = {}
-  %w[en fr].each do |meeting_lang|
-    next if ARGV[0] == '--fork' && fork
-
-    meeting_lang_sfx     = (meeting_lang == 'fr') ? "-fr" : ""
-    meeting_lang_sfx_dir = (meeting_lang == 'fr') ? "-fr" : "-en"
-
-    meetings = (meeting_lang == 'en') ? meetings_en : meetings_fr
-    publications = (meeting_lang == 'en') ? publications_en : publications_fr
-
-    index = {
-              "meetings" => {"fr" => [], "en" => []}, 
-              "decisions" => {"fr" => [], "en" => []},
-            }
-
-    meetings.css('.meetings-list__item').each do |meeting_div|
-      date = Bipm::Data::Importer::Common.extract_date(meeting_div.at_css('.meetings-list__informations-date').text)
-
-      title = meeting_div.at_css('.meetings-list__informations-title').text.strip
-      href = meeting_div.at_css('.meetings-list__informations-title').attr('href')
-      href = "/#{meeting_lang}" + href unless href.start_with? "/#{meeting_lang}/"
-
-      ident = href.split("/#{body}/").last.gsub('/', '.')
-      yr = href.include?("/wg/") ? nil : href.split('-').last
-      meeting_id = if href.include?("/wg/")
-        # workgroup logic
-        wg = href.split("/wg/").last.split("/").first
-        href.split("/").last
-      else
-        parts = href.split("/").last.split("-")
-        if parts.length == 2
-          # Only the number
-          parts[0]
-        else
-          parts[0] + parts[1].sub("_", "-")
-        end
-      end
-
-      meeting = VCR.use_cassette "#{body}/#{body}-meeting-#{ident}#{meeting_lang_sfx}" do
-        a.get href
-      end
-
-      # Duplicate data upstream:
-      # - https://www.bipm.org/en/committees/ci/cipm/104-_1-2015
-      # - https://www.bipm.org/en/committees/ci/cipm/104-_2-2015
-      yr = nil if [bodyid, meeting_id] == [:CIPM, '104-2']
-
-      # meeting logic, as in for the `meetings-xx` structure
-      # this structure contains recommendations
-
-      if yr
-        has_recommendations = false
-
-        pdf = Bipm::Data::Importer::Common.extract_pdf(meeting, meeting_lang)
-
-        h = {
-          "metadata" => {
-            "title" => title,
-            "identifier" => meeting_id,
-            "date" => date.to_s,
-            "source" => "BIPM - Pavillon de Breteuil",
-            "url" => meeting.uri.to_s,
-          }
-        }
-
-        h["pdf"] = pdf if pdf
-
-        resolutions = meeting.css(".bipm-resolutions .publications__content").map do |res_div|
-          res_div.at_css('a').attr('href')
-        end
-
-        # A mistake on a website, resolution 2 listed twice...
-        # https://www.bipm.org/fr/committees/ci/cipm/94-2005/
-        if [bodyid, meeting_lang, meeting_id] == [:CIPM, 'fr', '94'] && resolutions.sort.uniq != resolutions.sort
-          resolutions = (1..3).map do |i|
-            "https://www.bipm.org/fr/committees/ci/cipm/94-2005/resolution-#{i}"
-          end
-        end
-
-        h["resolutions"] = resolutions.map do |href|
-          href = href.gsub('/web/guest/', "/#{meeting_lang}/")
-          href = href.sub("www.bipm.org/", "www.bipm.org/#{meeting_lang}/") unless href.include? "/#{meeting_lang}/"
-
-          # error: https://www.bipm.org/fr/committees/ci/cipm/104-_1-2015 has wrong references to Recommandations
-          href = href.gsub('/104-2015/', '/104-_1-2015/')
-
-          res_id = href.split("-").last.to_i
-          res = VCR.use_cassette("#{body}/#{body}-recommendation-#{yr}-#{res_id}#{meeting_lang_sfx}") do
-            a.get href
-          end
-
-          has_recommendations = true
-
-          Bipm::Data::Importer::Common.parse_resolution(res, res_id, date, body, meeting_lang, "recommendation?")
-        end
-
-        index["meetings"][meeting_lang] << h
-      end
-
-      # decisions logic, as in for the `decisions-xx` structure
-      # this structure contains decisions
-
-      if meeting_id
-        h = {
-          "metadata" => {
-            "title" => title,
-            "identifier" => meeting_id,
-            "date" => date.to_s,
-            "source" => "BIPM - Pavillon de Breteuil",
-            "url" => meeting.uri.to_s
-          }
-        }
-
-        h["metadata"]["workgroup"] = wg if wg
-
-        h["resolutions"] = meeting.css('.bipm-decisions .decisions').map do |titletr|
-          title = titletr.at_css('.title-third').text.strip
-
-          type = case title
-          when /\AD[eé]cision/
-            "decision"
-          when /\AR[eé]solution/
-            "resolution"
-          when /\AAction/
-            "action"
-          when /\ARecomm[ae]ndation/
-            "recommendation"
-          else
-            "decision"
-          end
-
-          r = {
-            "dates" => [date.to_s],
-            "subject" => bodyid.to_s,
-            "type" => type,
-            "title" => title,
-            "identifier" => "#{titletr.attr('data-meeting')}-#{titletr.attr('data-number')}",
-            "url" => meeting.uri.to_s,
-            #TODO: "reference" => meeting.uri.merge(titletr.attr('data-link')).to_s,
-
-            "categories" => JSON.parse(titletr.attr('data-decisioncategories')).map(&:strip).uniq,
-
-            "considerations" => [],
-            "actions" => [],
-          }
-
-          contenttr = titletr.attr('data-text')
-          if contenttr.empty?
-            # This document has empty data-text attribute https://www.bipm.org/fr/committees/jc/jcrb/4-2000
-            contenttr = titletr.css('p').map(&:inner_html).join("\n")
-          end
-          contenttr = Nokogiri::HTML(contenttr)
-
-
-          Bipm::Data::Importer::Common.replace_links contenttr, meeting, meeting_lang
-
-          part = Bipm::Data::Importer::Common.ng_to_string(contenttr)
-          part = part.gsub(%r"<a name=\"haut\">(.*?)</a>"m, '\1')
-          parse = Nokogiri::HTML(part).text.strip
-
-          parse =~ /\A(((the|le|la|Le secrétaire du|Le président du|Les membres du|Le directeur du) +[BC][IG]PM( Director| President| Secretary| members)?|Dr May|W\.E\. May)[, ]+)/i
-          subject = $1
-          if subject
-            parse = parse[subject.length..-1]
-            part = part.gsub(/\A(<html><body>\n*<p>)#{Regexp.escape subject}/, '\1')
-            # Note: we set "CIPM" as the subject for all CIPM decisions
-            # r['subject'] = subject.strip
-          end
-
-          xparse = parse
-
-          (1..1024).each do |pass|
-            Bipm::Data::Importer::CONSIDERATIONS.any? do |k,v|
-              if xparse =~ /\A#{Bipm::Data::Importer::PREFIX}#{k}\b/i
-                r["considerations"] << {
-                  "type" => v,
-                  "date_effective" => date.to_s,
-                  "message" => Bipm::Data::Importer::Common.format_message(part),
-                }
-              end
-            end && break
-
-            Bipm::Data::Importer::ACTIONS.any? do |k,v|
-              if xparse =~ /\A#{Bipm::Data::Importer::PREFIX}#{k}\b/i
-                r["actions"] << {
-                  "type" => v,
-                  "date_effective" => date.to_s,
-                  "message" => Bipm::Data::Importer::Common.format_message(part),
-                }
-              end
-            end && break
-
-            case pass
-            when 1, 2, 3
-              xparse = xparse.gsub(/\A.*?(CIPM( President| in its meeting)?:?\n*|\((2018|CIPM)\)|de la 103e session) /m, '')
-            when 4
-              xparse = parse.gsub(/\A.*?, /m, '')
-            when 5
-              xparse = parse.gsub(/\A.*? (and|et) /m, '')
-            else
-              r["x-unparsed"] ||= []
-              r["x-unparsed"] << parse
-
-              break
-            end
-          end
-
-          r
-        end.sort_by { |i| [i["type"], i["identifier"].scan(/([0-9]+|[^0-9]+)/).map(&:first).map { |i| i =~ /[0-9]/ ? i.to_i : i }] }
-
-        # Mistake on a website: numbering is wrong
-        # https://www.bipm.org/fr/committees/jc/jcrb/43-2021
-        if [bodyid, meeting_lang, meeting_id] == [:JCRB, "fr", "43"]
-          ids = h["resolutions"].map { |i| [i["type"], i["identifier"]] }.sort
-
-          # Unfixed yet still, let's "massage" it
-          if ids != ids.uniq
-            idx = 1
-            h["resolutions"].each do |i|
-              next unless i["identifier"] == "43-1" && i["type"] == "action"
-              i["identifier"] += "-xxx-#{idx}"
-              idx += 1
-            end
-          end
-        end
-
-        h["working_documents"] = meeting.css('.portlet-boundary_CommitteePublications_ .publications__content').map do |d|
-          # Website depends on an Accept-Language header but caches responses. This creates non-determinism. We must unify those.
-          # $ curl -H "Accept-Language: " "https://www.bipm.org/en/committees/cc/ccqm/22-2016?$RANDOM$RANDOM" 2>/dev/null | grep -A2 publications__date
-          # <p class="publications__date">
-          #   Mon Jul 09 00:00:00 GMT 2018
-          # </p>
-          # $ curl -H "Accept-Language: en-us, en" "https://www.bipm.org/en/committees/cc/ccqm/22-2016?$RANDOM$RANDOM" 2>/dev/null | grep -A2 publications__date
-          # <p class="publications__date">
-          #   09/07/2018
-          # </p>
-          date_str = d.at_css('.publications__date')&.text&.strip
-          date_str = Date.parse(date_str).strftime("%d/%m/%Y") if date_str
-
-          {
-            "title" => d.at_css('.title-third').text.strip,
-            "pdf" => d.at_css('.title-third').attr('href'),
-            "description" => d.css('.publications__body').first.text.strip,
-            "author" => d.css('.publications__body').last.text.strip,
-            "date" => date_str,
-          }.compact
-        end
-
-        index["decisions"][meeting_lang] << h
-      end
-
-      # index logic: creates an index of all meetings mentioned on the website and
-      # their correct references to `decisions-xx` and `meetings-xx` if present.
-      # meeting = {
-      #   "title" => title,
-      #   "url" => href,
-      #   "ident" => ident,
-      #   "meeting_ref" => yr,
-      #   "decisions_ref" => meeting_id,
-      # }.compact
-
-      # index["meetings"] << meeting
-    end
-
-    # File.write("#{BASE_DIR}/cipm/index#{meeting_lang_sfx_dir}.yml", YAML.dump(index))
-
-    parts = index["meetings"][meeting_lang] + index["decisions"][meeting_lang]
-    parts = parts.group_by { |i| [i["metadata"]["workgroup"].to_s, i["metadata"]["identifier"].to_s] }
-    parts = parts.sort_by { |(wg,i),| [wg, i.to_i, i] }.to_h # Numeric sort by workgroup, then key
-
-    parts.each do |(wg, mid), hs|
-      h = {
-        "metadata" => hs.first["metadata"],
-        "pdf" => hs.first["pdf"],
-        "resolutions" => hs.map { |i| i["resolutions"] }.sum([]),
-        "working_documents" => hs.map { |i| i["working_documents"] || [] }.sum([])        
-      }
-
-      h.delete("pdf") unless h["pdf"]
-      h.delete("working_documents") if h["working_documents"].empty?
-
-      wg = hs.first["metadata"]["workgroup"]
-      if wg
-        fn = "#{BASE_DIR}/#{body}/workgroups/#{wg}/meetings#{meeting_lang_sfx_dir}/meeting-#{mid}.yml"
-      else
-        fn = "#{BASE_DIR}/#{body}/meetings#{meeting_lang_sfx_dir}/meeting-#{mid}.yml"
-      end
-
-      FileUtils.mkdir_p(File.dirname(fn))
-      File.write(fn, YAML.dump(h))
-    end
-
-    # Publications handling
-    categories = publications.css(".publications").map do |i|
-      title = i.previous_element&.text&.strip || "Misc"
-
-      {
-        "title" => title,
-        "documents" => i.css(".publications__content").map do |d|
-          {
-            "title" => d.at_css(".title-third").text.strip.gsub(/\s+/, ' '),
-            "pdf" => d.at_css(".title-third")&.attr("href")&.split('?')&.first,
-            # "description" => d.css('.publications__body')[0]&.text&.strip,
-            # "author" => d.css('.publications__body')[1]&.text&.strip,
-          }.compact
-        end
-      }
-    end
-
-    unless categories.empty?
-      doc = {
-        "metadata" => {
-          "url" => bodyurl.gsub("/en/", "/#{meeting_lang}/") + "/publications/"
-        },
-        "categories" => categories
-      }
-
-      fn = "#{BASE_DIR}/#{body}/publications-#{meeting_lang}.yml"
-
-      File.write(fn, YAML.dump(doc))
-    end
-
-    puts "* #{bodyid}/#{meeting_lang} parsing done"
-    exit! if ARGV[0] == '--fork'
-  end
-
-  2.times { Process.wait } if ARGV[0] == '--fork'
-  exit! if ARGV[0] == '--fork'
-end
-
-bodies.length.times { Process.wait } if ARGV[0] == '--fork'
+require "bipm-data-importer"
+exit Bipm::Data::Importer::CLI.new(ARGV).run

data/lib/bipm/data/importer/asciimath.rb

@@ -9,10 +9,10 @@ module Bipm
         SPACE_AFTER=/(?:\Z|(?= |\s|[,()\/.~"^]))/
 
         PREFIXES = /m|c|d|k|M|G|T|/
-        UNITS = /t|m|mol|cal|µ|s|g|W|cd|Hz|J|K|N|V|H|A|C|F|T|Wb|sr|lx|lm|bar|sb|h|rad|°C|°F|°K/
+        UNITS = /t|m|mol|cal|µ|s|g|W|cd|Hz|J|K|N|V|H|A|C|F|T|Wb|sr|lx|lm|bar|sb|h|rad|fm|°C|°F|°K/
 
         def asciidoc_extract_math str
-          str.gsub(/\b_(#{MATH}{1,3})_/, 'stem:[\1]')
+          str.gsub(/\b_?_(#{MATH}{1,3})_?_/, 'stem:[\1]')
             .gsub("_,_", ',') # Some mistake in formatting
             .gsub("^er^", 'ESCUPerESCUP') # French specialities
             .gsub(/(bar|A) (table|of|key|de|being|full|1)( |,)/, 'ESC\1 \2\3') # A is Ampere, but also a particle, bar is a bar but also a bar
@@ -69,4 +69,4 @@ module Bipm
       end
     end
   end
-end
+end

data/lib/bipm/data/importer/bodies.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Bipm
+  module Data
+    module Importer
+      # Open/closed registry of all known BIPM bodies.
+      # Adding a body = adding a row here, not editing CLI/scraper code.
+      module Bodies
+        ALL = {
+          jcrb:  Body.new(id: :jcrb,  display_name: "JCRB",  path: "committees/jc/jcrb",                  strategy: :spa_meetings),
+          jcgm:  Body.new(id: :jcgm,  display_name: "JCGM",  path: "committees/jc/jcgm",                  strategy: :spa_meetings),
+          ccu:   Body.new(id: :ccu,   display_name: "CCU",   path: "committees/cc/ccu",                   strategy: :spa_meetings),
+          cctf:  Body.new(id: :cctf,  display_name: "CCTF",  path: "committees/cc/cctf",                  strategy: :spa_meetings),
+          cct:   Body.new(id: :cct,   display_name: "CCT",   path: "committees/cc/cct",                   strategy: :spa_meetings),
+          ccri:  Body.new(id: :ccri,  display_name: "CCRI",  path: "committees/cc/ccri",                  strategy: :spa_meetings),
+          ccpr:  Body.new(id: :ccpr,  display_name: "CCPR",  path: "committees/cc/ccpr",                  strategy: :spa_meetings),
+          ccqm:  Body.new(id: :ccqm,  display_name: "CCQM",  path: "committees/cc/ccqm",                  strategy: :spa_meetings),
+          ccm:   Body.new(id: :ccm,   display_name: "CCM",   path: "committees/cc/ccm",                   strategy: :spa_meetings),
+          ccl:   Body.new(id: :ccl,   display_name: "CCL",   path: "committees/cc/ccl",                   strategy: :spa_meetings),
+          ccem:  Body.new(id: :ccem,  display_name: "CCEM",  path: "committees/cc/ccem",                  strategy: :spa_meetings),
+          ccauv: Body.new(id: :ccauv, display_name: "CCAUV", path: "committees/cc/ccauv",                 strategy: :spa_meetings),
+          cipm:  Body.new(id: :cipm,  display_name: "CIPM",  path: "committees/ci/cipm",                  strategy: :spa_meetings),
+          cgpm:  Body.new(id: :cgpm,  display_name: "CGPM",  path: "worldwide-metrology/cgpm/resolutions.html", strategy: :static_index),
+        }.freeze
+
+        UnknownBodyError = Class.new(ArgumentError)
+
+        def self.all
+          ALL.values
+        end
+
+        def self.find(id)
+          key = id.to_sym
+          ALL[key] || raise(UnknownBodyError, "unknown body: #{id.inspect} (known: #{ALL.keys.inspect})")
+        end
+        singleton_class.alias_method :[], :find
+
+        def self.each(&block)
+          ALL.values.each(&block)
+        end
+        singleton_class.include Enumerable
+      end
+    end
+  end
+end

data/lib/bipm/data/importer/body.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Bipm
+  module Data
+    module Importer
+      # A BIPM committee body (CIPM, CGPM, CCAUV, etc.).
+      #
+      # `path` is locale-agnostic — `url(:en)` and `url(:fr)` are constructed
+      # symmetrically by inserting the language segment into the BIPM URL
+      # template. Neither language is canonical; both are first-class.
+      class Body
+        BIPM_ORIGIN = "https://www.bipm.org".freeze
+
+        attr_reader :id, :display_name, :path, :strategy
+
+        def initialize(id:, display_name:, path:, strategy:)
+          @id = id
+          @display_name = display_name
+          @path = path
+          @strategy = strategy
+        end
+
+        def url(language)
+          lang = language.is_a?(Language) ? language : Language.find(language)
+          "#{BIPM_ORIGIN}/#{lang}/#{path}"
+        end
+
+        def eql?(other)
+          other.is_a?(Body) && @id == other.id
+        end
+        alias == eql?
+
+        def hash
+          @id.hash
+        end
+
+        def to_s
+          "#{display_name} (#{id})"
+        end
+      end
+    end
+  end
+end

data/lib/bipm/data/importer/cgpm.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Bipm
+  module Data
+    module Importer
+      # Backward-compatibility facade. The implementation lives in
+      # Bipm::Data::Importer::Strategies::StaticIndex; this module exists
+      # so that older code calling CGPM.run(agent, base_dir) keeps working.
+      #
+      # New code should go through Bipm::Data::Importer.fetch(:cgpm).
+      module CGPM
+        def self.run(agent, base_dir)
+          Strategies::StaticIndex.new(
+            body: Bodies[:cgpm],
+            base_dir: base_dir,
+          ).call(agent: agent)
+        end
+      end
+    end
+  end
+end