bipm-data-importer 0.2.2 → 0.4.0

data/lib/bipm/data/importer/clauses.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+# Single source of truth for the clause taxonomy used by every scraping
+# strategy. Replaces the previously-duplicated CONSIDERATIONS/ACTIONS hashes
+# in common.rb and the VERB_TO_TYPE/FR_VERB_TO_EN/CONSIDERATION_VERBS/
+# ACTION_VERBS constants in cgpm.rb.
+#
+# Design principle: French and English are first-class citizens. The VERBS
+# table is keyed by either language's verb form and maps directly to a
+# (type, category) pair — there is no "translate FR → EN then look up"
+# step. Adding a new language means adding new keys to the same table.
+module Bipm
+  module Data
+    module Importer
+      module Clauses
+        CATEGORY_CONSIDERATION = :consideration
+        CATEGORY_ACTION = :action
+
+        # Phrase-level regex patterns. Order matters: tried top-to-bottom,
+        # first match wins. Two hashes preserved (rather than one merged
+        # table) so that consumers can ask "is this text a consideration?"
+        # without scanning action patterns. Patterns themselves freely mix
+        # English and French alternatives — no language is canonical.
+        CONSIDERATIONS = {
+          /(?:having(?: regard)?|ayant|concerne|vu la|agissant conformément|sachant|de porter)/i => "having / having regard",
+          /(?:noting|to note|took note|note[sd]?|taking note|takes note|constatant|constate|that|notant|notant que|note également|(?:prend|prenant) (?:acte|note))/i => "noting",
+          /(?:recognizing|recognizes|reconnaissant|reconnaît)/i => "recognizing",
+          /(?:acknowledging|admet|entendu|(?:and |)aware that|anticipa(?:ting|nt))/i => "acknowledging",
+          /(?:(?:further )?recall(?:ing|s|ed)|rappelant|rappelle|rappelantla)/i => "recalling / further recalling",
+          /(?:re-?affirm(?:ing|s)|réaffirme)/i => "reaffirming",
+          /(?:consid(?:er(?:ing|)|érant|ère|ers|ered|érantque|érantle)|après examen|estime|is of the opinion|examinera|en raison|by reason)/i => "considering",
+          /(?:The Consultative Committee for Time and Frequency \(CCTF\), at|Le Comité consultatif du temps et des fréquences \(CCTF\), à)/i => "considering",
+          /(?:taking into account|(prend|prenant) en considération|taking into consideration|tenant compte|envisager)/i => "taking into account",
+          "pursuant to" => "pursuant to",
+          /(?:bearing in mind)/i => "bearing in mind",
+          /(?:emphasizing|soulignant)/i => "emphasizing",
+          "concerned" => "concerned",
+          /(?:accept(?:s|ed|ing|e)|acceptant)/i => "accepts",
+          /(?:observing|observant que|r[ée]ali(?:zing|sant))/i => "observing",
+          /(?:referring|se référant)/i => "referring",
+          /(?:acting in accordance|agissant conformément|conformément)/i => "acting",
+          /(?:empowered by|habilité par)/i => "empowers",
+        }.freeze
+
+        ACTIONS = {
+          /(?:adopts|adopt[eé]d?|convient d'adopter)/ => "adopts",
+          /(?:thanks|thanked|expresse[sd](?:[ -]| its )appreciation|appréciant|pays tribute|rend hommage|remercie|support(?:ed|s))/i => "thanks / expresses-appreciation",
+          /(?:approu?v[eé][ds]?|approuv[ae]nt|approving|entérine|(?:It was )?agree[sd]?|soutient|exprime son accord|n'est pas d'accord|convient)/i => "approves",
+          /(?:d[eé]cid(?:e[ds]?|é)|ratifies?|r[eé]vised?)/i => "decides",
+          /(?:d[ée]clares?|d[ée]finition)/i => "declares",
+          /(?:The unit of length is|Supplementary units|Derived units|Principl?es|Les Délégués des États|Les v(?:œ|\u{9C})ux ou propositions)/i => "declares",
+          /(?:L'unité de longueur|Unités supplémentaires|Unités dérivées|(?:\*_)?New candle|(?:\*_)?New lumen|(?:A\) )?D[ée]finitions (?:of|des)|Cubic decimetre|Clarification of|Revision of)/i => "declares",
+          /(?:Unit of force|Définitions des|Décimètre cube|Étalons secondaires|Unité spéciale|Efficacités lumineuses|From three names|Entre les trois termes)/i => "declares",
+          /(?:Unité de force|(?:Joule|Watt|Volt|Ohm|Amp[eè]re|Coulomb|Farad|Henry|Weber) \(unité?|Bougie nouvelle|Lumen nouveau|announces that|annonce que)/i => "declares",
+          /(?:Les unités photométriques|\(A\) D[eé]finitions|The photometric units|will (?:provide|circulate|issue|identify|notify|contact|review))/i => "declares",
+          /(?:Appendix 1 of the|L'Annexe 1 de la|increased|a (?:examiné|préparé)|transmettra|fournira|increased|developed a document|prendra contact)/i => "declares",
+          /(?:Le Temps Atomique International |International Atomic Time \(TAI\) |will meet )/i => "declares",
+          /(?:ask[s ]|asked|souhaite|souhaiterait)/i => "asks",
+          /(?:(?:further |et )?invit(?:[ée][ds]?|era)|renouvelle en conséquence|convient d'inviter)/i => "invites / further invites",
+          /(?:resolve[sd]?)/i => "resolves",
+          /(?:confirms|confirmed?|confirme que|committed|s'engageant)/i => "confirms",
+          /(?:welcom(?:e[sd]?|ing)|accueille favorablement(?:les)?|salu(?:e|ant))/i => "welcomes",
+          /(?:recomm(?:ends?|ande(?:nt|)|ended)|endorsed|LISTE DES RADIATIONS|1 Radiations recommandées|LIST OF RECOMMENDED|1 Recommended radiations|aim(?:s|ing)|a pour objectif|should)/i => "recommends",
+          /(?:requests?|requested|demande(?:ra)?|requi[eè]r(?:en|)t|must)|l'intention d’examiner/ => "requests",
+          /(?:(?:is |are |)(?:to |)(?:re-?|)(?:amend|investigate|delete|help|present|develop|create|refer|add|formalise|update|collaborate|ensure|modify|prepare|look|report|consider|continue|make|bring|post|request|draw|raise|draft|circulate|arrange|provide|send|write|check|amend|forward|distribute|pursue|inform|coordinate|discuss|submit|ask|inquire|put)|will)/i => "requests",
+          /(?:congratulate[sd]?|félicite)/i => "congratulates",
+          /(?:instructs|instructed|inform[es]|intends to)/i => "instructs",
+          /(?:(?:strongly |)urges|prie instamment)/i => "urges",
+          /(?:appoints|(?:re)?appointed|granted|reconduit|commended|accorde)/i => "appoints",
+          /(?:donn(?:e|ées)|Pendant la période|voted|established a \w+ task group)/i => "appoints",
+          /(?:convient d'éablir|transfère|confie|établit|Étant donné que trois sièges|As there will be three vacancies)/i => "appoints",
+          /(?:La Recommandation 1 du Groupe|Recommendation 1 of the ad hoc)/i => "appoints",
+          /(?:élit|nomme|elected|nominated)/ => "elects",
+          /(?:gave the \w+ \w+ the authority|autorise|authorized)/ => "authorizes",
+          /(?:charged?)/ => "charges",
+          /(?:resolve[sd]? further)/i => "resolves further",
+          /(?:calls upon|draws the attention|attire l'attention|lance un appel|called upon)/i => "calls upon",
+          /(?:encourages?d?|espère|propose[ds]?)/i => "encourages",
+          /(?:affirms|reaffirming|réaffirmant|concurs)/i => "affirms / reaffirming",
+          /(?:states)/i => "states",
+          /(?:remarks|remarques)/i => "remarks",
+          /(?:judges)/i => "judges",
+          /(?:sanction(?:s|né?e))/i => "sanctions",
+          /(?:abrogates|abroge)/i => "abrogates",
+          /(?:empowers|habilite)/i => "empowers",
+        }.freeze
+
+        # Verb → (type, category). Symmetric across English and French —
+        # both languages are first-class keys, no canonical form.
+        VERBS = {
+          # consideration verbs
+          "having"         => ["having / having regard", CATEGORY_CONSIDERATION],
+          "ayant"          => ["having / having regard", CATEGORY_CONSIDERATION],
+          "vu"             => ["having / having regard", CATEGORY_CONSIDERATION],
+          "considering"    => ["considering",            CATEGORY_CONSIDERATION],
+          "considérant"    => ["considering",            CATEGORY_CONSIDERATION],
+          "noting"         => ["noting",                 CATEGORY_CONSIDERATION],
+          "notant"         => ["noting",                 CATEGORY_CONSIDERATION],
+          "notes"          => ["notes",                  CATEGORY_CONSIDERATION],
+          "note"           => ["notes",                  CATEGORY_CONSIDERATION],
+          "recognizing"    => ["recognizing",            CATEGORY_CONSIDERATION],
+          "reconnaissant"  => ["recognizing",            CATEGORY_CONSIDERATION],
+          "reconnaît"      => ["recognizing",            CATEGORY_CONSIDERATION],
+          "reaffirming"    => ["reaffirming",            CATEGORY_CONSIDERATION],
+          "réaffirmant"    => ["reaffirming",            CATEGORY_CONSIDERATION],
+          "recalling"      => ["recalling / further recalling", CATEGORY_CONSIDERATION],
+          "rappelant"      => ["recalling / further recalling", CATEGORY_CONSIDERATION],
+          "rappelle"       => ["recalling / further recalling", CATEGORY_CONSIDERATION],
+          "acknowledging"  => ["acknowledging",          CATEGORY_CONSIDERATION],
+          "taking"         => ["taking into account",    CATEGORY_CONSIDERATION],
+          "tenant"         => ["taking into account",    CATEGORY_CONSIDERATION],
+          "pursuant"       => ["pursuant to",            CATEGORY_CONSIDERATION],
+          "bearing"        => ["bearing in mind",        CATEGORY_CONSIDERATION],
+          "emphasizing"    => ["emphasizing",            CATEGORY_CONSIDERATION],
+          "soulignant"     => ["emphasizing",            CATEGORY_CONSIDERATION],
+          "concerned"      => ["concerned",              CATEGORY_CONSIDERATION],
+          "concerné"       => ["concerned",              CATEGORY_CONSIDERATION],
+          "accepts"        => ["accepts",                CATEGORY_CONSIDERATION],
+          "accepte"        => ["accepts",                CATEGORY_CONSIDERATION],
+          "observing"      => ["observing",              CATEGORY_CONSIDERATION],
+          "observant"      => ["observing",              CATEGORY_CONSIDERATION],
+          "réalise"        => ["observing",              CATEGORY_CONSIDERATION],
+          "referring"      => ["referring",              CATEGORY_CONSIDERATION],
+          "se référant"    => ["referring",              CATEGORY_CONSIDERATION],
+          "acting"         => ["acting",                 CATEGORY_CONSIDERATION],
+          "agissant"       => ["acting",                 CATEGORY_CONSIDERATION],
+          "empowers"       => ["empowers",               CATEGORY_CONSIDERATION],
+          "habilité"       => ["empowers",               CATEGORY_CONSIDERATION],
+          "habilite"       => ["empowers",               CATEGORY_CONSIDERATION],
+          # action verbs
+          "decides"        => ["decides",                CATEGORY_ACTION],
+          "décide"         => ["decides",                CATEGORY_ACTION],
+          "declares"       => ["declares",               CATEGORY_ACTION],
+          "déclare"        => ["declares",               CATEGORY_ACTION],
+          "invites"        => ["invites / further invites", CATEGORY_ACTION],
+          "invite"         => ["invites / further invites", CATEGORY_ACTION],
+          "resolves"       => ["resolves",               CATEGORY_ACTION],
+          "résout"         => ["resolves",               CATEGORY_ACTION],
+          "confirms"       => ["confirms",               CATEGORY_ACTION],
+          "confirme"       => ["confirms",               CATEGORY_ACTION],
+          "welcomes"       => ["welcomes",               CATEGORY_ACTION],
+          "accueille"      => ["welcomes",               CATEGORY_ACTION],
+          "recommends"     => ["recommends",             CATEGORY_ACTION],
+          "recommande"     => ["recommends",             CATEGORY_ACTION],
+          "requests"       => ["requests",               CATEGORY_ACTION],
+          "demande"        => ["requests",               CATEGORY_ACTION],
+          "appoints"       => ["appoints",               CATEGORY_ACTION],
+          "nomme"          => ["appoints",               CATEGORY_ACTION],
+          "encourages"     => ["encourages",             CATEGORY_ACTION],
+          "encourage"      => ["encourages",             CATEGORY_ACTION],
+          "affirms"        => ["affirms / reaffirming",  CATEGORY_ACTION],
+          "affirme"        => ["affirms / reaffirming",  CATEGORY_ACTION],
+          "calls"          => ["calls upon",             CATEGORY_ACTION],
+          "lance un appel" => ["calls upon",             CATEGORY_ACTION],
+          "states"         => ["states",                 CATEGORY_ACTION],
+          "indique"        => ["states",                 CATEGORY_ACTION],
+          "remarks"        => ["remarks",                CATEGORY_ACTION],
+          "remarques"      => ["remarks",                CATEGORY_ACTION],
+          "urges"          => ["urges",                  CATEGORY_ACTION],
+          "prie"           => ["urges",                  CATEGORY_ACTION],
+          "instructs"      => ["instructs",              CATEGORY_ACTION],
+          "informe"        => ["instructs",              CATEGORY_ACTION],
+          "adopts"         => ["adopts",                 CATEGORY_ACTION],
+          "adopte"         => ["adopts",                 CATEGORY_ACTION],
+          "thanks"         => ["thanks / expresses-appreciation", CATEGORY_ACTION],
+          "remercie"       => ["thanks / expresses-appreciation", CATEGORY_ACTION],
+          "approves"       => ["approves",               CATEGORY_ACTION],
+          "approuve"       => ["approves",               CATEGORY_ACTION],
+          "asks"           => ["asks",                   CATEGORY_ACTION],
+          "souhaite"       => ["asks",                   CATEGORY_ACTION],
+          "congratulates"  => ["congratulates",          CATEGORY_ACTION],
+          "félicite"       => ["congratulates",          CATEGORY_ACTION],
+          "elects"         => ["elects",                 CATEGORY_ACTION],
+          "élit"           => ["elects",                 CATEGORY_ACTION],
+          "authorizes"     => ["authorizes",             CATEGORY_ACTION],
+          "autorise"       => ["authorizes",             CATEGORY_ACTION],
+          "charges"        => ["charges",                CATEGORY_ACTION],
+          "judges"         => ["judges",                 CATEGORY_ACTION],
+          "juge"           => ["judges",                 CATEGORY_ACTION],
+          "sanctions"      => ["sanctions",              CATEGORY_ACTION],
+          "sanctionne"     => ["sanctions",              CATEGORY_ACTION],
+          "abrogates"      => ["abrogates",              CATEGORY_ACTION],
+          "abroge"         => ["abrogates",              CATEGORY_ACTION],
+        }.freeze
+
+        CONSIDERATION_TYPES = CONSIDERATIONS.values.uniq.freeze
+        ACTION_TYPES = ACTIONS.values.uniq.freeze
+
+        PREFIX1 = /(?:The|Le) CIPM |La Conférence |M. Volterra |M. le Président |unanimously |would |a |sont |will |were |did not |strongly |(?:La|The) (?:\d+(?:e|th)|Quinzième) Conférence Générale des Poids et Mesures(?: a |,\s+)?/i
+        PREFIX2 = /The \d+th Conférence Générale des Poids et Mesures |The Conference |and |et (?:en |)|has |renouvelle sa |renews its |further |and further |En ce qui |après avoir |\.\.\.\n+\t*/i
+        PREFIX3 = /Sur la proposition de M. le Président, la convocation de cette Conférence de Thermométrie est |Le texte corrigé, finalement |(?:The|Le) Comité International(?: des Poids et Mesures)?(?: \(CIPM\))?(?: a |,)?\s*/i
+        PREFIX4 = /(?:The |Le |)(?:JCRB|JCGM|CCU|CCTF|CCT|CCRI|CCPR|CCQM|CCM|CCL|CCEM|CCAUV|KCDB),? (?:also |)|Each RMO |fully |The JCRB Rules of Procedure are |Bob Watters and Claudine Thomas /
+        PREFIX5 = /(?:The |Le |All |)(?:incoming |)(?:JCRB |KCDB |)(?:documents|(?:Consultative |)Committees?|Office|Chairman(?: and Secretary|)|Joint BIPM[\/-]ILAC Working Group(?: \(see Action 22\))|RMO(?:[- ]JCRB|) Representatives(?: to the JRCB|)|(?:BIPM |)Director(?: of BIPM|)|SIM|(?:Exec(?:utive|) |)Secretary(?:\(ies\)|)|RMOs, except SIM,|RMOs|APMP|\(?(?:[MD]r|Prof) [A-Z][a-zR-]+\)?|CMCs|EUR[AO]MET|COOMET|GULFMET) |It was /
+        PREFIX6 = /“|"|« à |All RMO documents related to review procedures |Mr Lam and Dr Kühne |The Prof. Kühne, Mr Jones and the Executive Secretary |Ajchara Charoensook, from APMP, /
+
+        PREFIX = /(?:#{PREFIX1}|#{PREFIX2}|#{PREFIX3}|#{PREFIX4}|#{PREFIX5}|#{PREFIX6})?/i
+
+        SUFFIX = / (?:that|que)\b|(?: (?:the |that |le |que les )?((?:[A-Z]|national|laboratoires).{0,80}?)(?: to)?\b|)/
+
+        LEADING_CONJUNCTIONS = /\A(?:et|and|further|de plus|puis|also)\s+/.freeze
+        TRAILING_PUNCTUATION = /[.:,;]\z/.freeze
+
+        # Strip leading conjunctions and trailing punctuation from a verb token.
+        # Returns the verb in whichever language it was supplied — does NOT
+        # translate between French and English. Use lookup_verb to find the
+        # canonical (type, category) for either language's verb.
+        def self.normalize_verb(raw)
+          raw.to_s.downcase.strip
+             .sub(TRAILING_PUNCTUATION, "")
+             .strip
+             .sub(LEADING_CONJUNCTIONS, "")
+             .strip
+        end
+
+        # Lookup a normalized verb's (type, category). Verb may be in either
+        # English or French — VERBS table has both as first-class keys.
+        def self.lookup_verb(verb)
+          VERBS.fetch(verb) { [fallback_type(verb), fallback_category(verb)] }
+        end
+
+        def self.category_for_type(type)
+          return CATEGORY_CONSIDERATION if CONSIDERATION_TYPES.include?(type)
+          return CATEGORY_ACTION if ACTION_TYPES.include?(type)
+          CATEGORY_ACTION
+        end
+
+        def self.fallback_type(verb)
+          return "considering" if verb == "having" || verb == "ayant" || verb == "vu"
+          "decides"
+        end
+
+        def self.fallback_category(verb)
+          return CATEGORY_CONSIDERATION if verb == "having" || verb == "ayant" || verb == "vu"
+          CATEGORY_ACTION
+        end
+      end
+    end
+  end
+end

data/lib/bipm/data/importer/cli.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module Bipm
+  module Data
+    module Importer
+      # Argument parsing and dispatch. The exe/bipm-fetch shim delegates here
+      # so the executable stays a one-liner and the CLI is fully testable.
+      class CLI
+        Options = Struct.new(:body_id, :languages, :base_dir, :fork)
+
+        def initialize(argv)
+          @argv = argv
+        end
+
+        def run
+          opts = parse(@argv)
+          if opts.body_id
+            run_one(opts)
+          else
+            run_all(opts)
+          end
+          0
+        rescue Bodies::UnknownBodyError => e
+          warn "error: #{e.message}"
+          2
+        rescue => e
+          warn "fatal: #{e.class}: #{e.message}"
+          warn e.backtrace.first(5).join("\n")
+          1
+        end
+
+        def parse(argv)
+          opts = Options.new(nil, nil, "data", false)
+          parser = OptionParser.new do |p|
+            p.banner = "Usage: bipm-fetch [options]"
+            p.on("--body=ID", "Fetch a single body (e.g. cgpm, cipm)") { |v| opts.body_id = v.to_sym }
+            p.on("--language=LANG", "Restrict to one language (en or fr)") do |v|
+              opts.languages = [Language.find(v)]
+            end
+            p.on("--base-dir=DIR", "Output directory (default: data)") { |v| opts.base_dir = v }
+            p.on("--fork", "Fork one process per body") { opts.fork = true }
+          end
+          parser.parse!(argv.dup)
+          opts
+        end
+
+        private
+
+        def run_one(opts)
+          fetcher = Fetcher.new(
+            body: opts.body_id,
+            languages: opts.languages || Language.all,
+            base_dir: opts.base_dir,
+          )
+          fetcher.call
+        end
+
+        def run_all(opts)
+          languages = opts.languages || Language.all
+          fetchers = Bodies.all.map do |body|
+            Fetcher.new(body: body, languages: languages, base_dir: opts.base_dir)
+          end
+          if opts.fork
+            run_forked(fetchers)
+          else
+            fetchers.each(&:call)
+          end
+        end
+
+        def run_forked(fetchers)
+          pids = []
+          fetchers.each do |f|
+            pid = fork { f.call; exit! }
+            pids << pid if pid
+          end
+          pids.each { |pid| Process.wait(pid) }
+        end
+      end
+    end
+  end
+end

data/lib/bipm/data/importer/common.rb

@@ -1,93 +1,36 @@
+# frozen_string_literal: true
+
 require "mechanize"
+require "nokogiri"
 require "coradoc/input/html"
 require "vcr"
 require "date"
 require "fileutils"
-require "pry"
-require_relative "asciimath"
 
 VCR.configure do |c|
   c.cassette_library_dir = __dir__ + "/../../../../cassettes"
   c.hook_into :webmock
+  # Default: replay existing cassettes (deterministic, no network).
+  # Set BIPM_VCR_RECORD=all to overwrite cassettes with fresh responses
+  # from bipm.org; set BIPM_VCR_RECORD=new_episodes to record only URLs
+  # not already in a cassette. VCR's :once (default) raises if a request
+  # is missing from its cassette.
+  if ENV["BIPM_VCR_RECORD"]
+    c.default_cassette_options = { record: ENV["BIPM_VCR_RECORD"].to_sym }
+  end
 end
 
 module Bipm
   module Data
     module Importer
-      CONSIDERATIONS = {
-        /(?:having(?: regard)?|ayant|concerne|vu la|agissant conformément|sachant|de porter)/i => "having / having regard",
-        /(?:noting|to note|took note|note[sd]?|taking note|takes note|constatant|constate|that|notant|notant que|note également|(?:prend|prenant) (?:acte|note))/i => "noting",
-        /(?:recognizing|recognizes|reconnaissant|reconnaît)/i => "recognizing",
-        /(?:acknowledging|admet|entendu|(?:and |)aware that|anticipa(?:ting|nt))/i => "acknowledging",
-        /(?:(?:further )?recall(?:ing|s|ed)|rappelant|rappelle|rappelantla)/i => "recalling / further recalling",
-        /(?:re-?affirm(?:ing|s)|réaffirme)/i => "reaffirming",
-        /(?:consid(?:er(?:ing|)|érant|ère|ers|ered|érantque|érantle)|après examen|estime|is of the opinion|examinera|en raison|by reason)/i => "considering",
-        /(?:The Consultative Committee for Time and Frequency \(CCTF\), at|Le Comité consultatif du temps et des fréquences \(CCTF\), à)/i => "considering",
-        /(?:taking into account|(prend|prenant) en considération|taking into consideration|tenant compte|envisager)/i => "taking into account",
-        "pursuant to" => "pursuant to",
-        /(?:bearing in mind)/i => "bearing in mind",
-        /(?:emphasizing|soulignant)/i => "emphasizing",
-        "concerned" => "concerned",
-        /(?:accept(?:s|ed|ing|e)|acceptant)/i => "accepts",
-        /(?:observing|observant que|r[ée]ali(?:zing|sant))/i => "observing",
-        /(?:referring|se référant)/i => "referring",
-        /(?:acting in accordance|agissant conformément|conformément)/i => "acting",
-        /(?:empowered by|habilité par)/i => "empowers",
-      }
-
-      ACTIONS = {
-        /(?:adopts|adopt[eé]d?|convient d'adopter)/ => "adopts",
-        /(?:thanks|thanked|expresse[sd](?:[ -]| its )appreciation|appréciant|pays tribute|rend hommage|remercie|support(?:ed|s))/i => "thanks / expresses-appreciation",
-        /(?:approu?v[eé][ds]?|approuv[ae]nt|approving|entérine|(?:It was )?agree[sd]?|soutient|exprime son accord|n'est pas d'accord|convient)/i => "approves",
-        /(?:d[eé]cid(?:e[ds]?|é)|ratifies?|r[eé]vised?)/i => "decides",
-        /(?:d[ée]clares?|d[ée]finition)/i => "declares",
-        /(?:The unit of length is|Supplementary units|Derived units|Principl?es|Les Délégués des États|Les v(?:œ|\u{9C})ux ou propositions)/i => "declares",
-        /(?:L'unité de longueur|Unités supplémentaires|Unités dérivées|(?:\*_)?New candle|(?:\*_)?New lumen|(?:A\) )?D[ée]finitions (?:of|des)|Cubic decimetre|Clarification of|Revision of)/i => "declares",
-        /(?:Unit of force|Définitions des|Décimètre cube|Étalons secondaires|Unité spéciale|Efficacités lumineuses|From three names|Entre les trois termes)/i => "declares",
-        /(?:Unité de force|(?:Joule|Watt|Volt|Ohm|Amp[eè]re|Coulomb|Farad|Henry|Weber) \(unité?|Bougie nouvelle|Lumen nouveau|announces that|annonce que)/i => "declares",
-        /(?:Les unités photométriques|\(A\) D[eé]finitions|The photometric units|will (?:provide|circulate|issue|identify|notify|contact|review))/i => "declares",
-        /(?:Appendix 1 of the|L'Annexe 1 de la|increased|a (?:examiné|préparé)|transmettra|fournira|increased|developed a document|prendra contact)/i => "declares",
-        /(?:Le Temps Atomique International |International Atomic Time \(TAI\) |will meet )/i => "declares",
-        /(?:ask[s ]|asked|souhaite|souhaiterait)/i => "asks",
-        /(?:(?:further |et )?invit(?:[ée][ds]?|era)|renouvelle en conséquence|convient d'inviter)/i => "invites / further invites",
-        /(?:resolve[sd]?)/i => "resolves",
-        /(?:confirms|confirmed?|confirme que|committed|s'engageant)/i => "confirms",
-        /(?:welcom(?:e[sd]?|ing)|accueille favorablement(?:les)?|salu(?:e|ant))/i => "welcomes",
-        /(?:recomm(?:ends?|ande(?:nt|)|ended)|endorsed|LISTE DES RADIATIONS|1 Radiations recommandées|LIST OF RECOMMENDED|1 Recommended radiations|aim(?:s|ing)|a pour objectif|should)/i => "recommends",
-        /(?:requests?|requested|demande(?:ra)?|requi[eè]r(?:en|)t|must)|l'intention d’examiner/ => "requests",
-        /(?:(?:is |are |)(?:to |)(?:re-?|)(?:amend|investigate|delete|help|present|develop|create|refer|add|formalise|update|collaborate|ensure|modify|prepare|look|report|consider|continue|make|bring|post|request|draw|raise|draft|circulate|arrange|provide|send|write|check|amend|forward|distribute|pursue|inform|coordinate|discuss|submit|ask|inquire|put)|will)/i => "requests",
-        /(?:congratulate[sd]?|félicite)/i => "congratulates",
-        /(?:instructs|instructed|inform[es]|intends to)/i => "instructs",
-        /(?:(?:strongly |)urges|prie instamment)/i => "urges",
-        /(?:appoints|(?:re)?appointed|granted|reconduit|commended|accorde)/i => "appoints",
-        /(?:donn(?:e|ées)|Pendant la période|voted|established a \w+ task group)/i => "appoints",
-        /(?:convient d'éablir|transfère|confie|établit|Étant donné que trois sièges|As there will be three vacancies)/i => "appoints",
-        /(?:La Recommandation 1 du Groupe|Recommendation 1 of the ad hoc)/i => "appoints",
-        /(?:élit|nomme|elected|nominated)/ => "elects",
-        /(?:gave the \w+ \w+ the authority|autorise|authorized)/ => "authorizes",
-        /(?:charged?)/ => "charges",
-        /(?:resolve[sd]? further)/i => "resolves further",
-        /(?:calls upon|draws the attention|attire l'attention|lance un appel|called upon)/i => "calls upon",
-        /(?:encourages?d?|espère|propose[ds]?)/i => "encourages",
-        /(?:affirms|reaffirming|réaffirmant|concurs)/i => "affirms / reaffirming",
-        /(?:states)/i => "states",
-        /(?:remarks|remarques)/i => "remarks",
-        /(?:judges)/i => "judges",
-        /(?:sanction(?:s|né?e))/i => "sanctions",
-        /(?:abrogates|abroge)/i => "abrogates",
-        /(?:empowers|habilite)/i => "empowers",
-      }
-
-      PREFIX1 = /(?:The|Le) CIPM |La Conférence |M. Volterra |M. le Président |unanimously |would |a |sont |will |were |did not |strongly |(?:La|The) (?:\d+(?:e|th)|Quinzième) Conférence Générale des Poids et Mesures(?: a |,\s+)?/i
-      PREFIX2 = /The \d+th Conférence Générale des Poids et Mesures |The Conference |and |et (?:en |)|has |renouvelle sa |renews its |further |and further |En ce qui |après avoir |\.\.\.\n+\t*/i
-      PREFIX3 = /Sur la proposition de M. le Président, la convocation de cette Conférence de Thermométrie est |Le texte corrigé, finalement |(?:The|Le) Comité International(?: des Poids et Mesures)?(?: \(CIPM\))?(?: a |,)?\s*/i
-      PREFIX4 = /(?:The |Le |)(?:JCRB|JCGM|CCU|CCTF|CCT|CCRI|CCPR|CCQM|CCM|CCL|CCEM|CCAUV|KCDB),? (?:also |)|Each RMO |fully |The JCRB Rules of Procedure are |Bob Watters and Claudine Thomas /
-      PREFIX5 = /(?:The |Le |All |)(?:incoming |)(?:JCRB |KCDB |)(?:documents|(?:Consultative |)Committees?|Office|Chairman(?: and Secretary|)|Joint BIPM[\/-]ILAC Working Group(?: \(see Action 22\))|RMO(?:[- ]JCRB|) Representatives(?: to the JRCB|)|(?:BIPM |)Director(?: of BIPM|)|SIM|(?:Exec(?:utive|) |)Secretary(?:\(ies\)|)|RMOs, except SIM,|RMOs|APMP|\(?(?:[MD]r|Prof) [A-Z][a-zR-]+\)?|CMCs|EUR[AO]MET|COOMET|GULFMET) |It was /
-      PREFIX6 = /“|"|« à |All RMO documents related to review procedures |Mr Lam and Dr Kühne |The Prof. Kühne, Mr Jones and the Executive Secretary |Ajchara Charoensook, from APMP, /
-
-      PREFIX = /(?:#{PREFIX1}|#{PREFIX2}|#{PREFIX3}|#{PREFIX4}|#{PREFIX5}|#{PREFIX6})?/i
-
-      SUFFIX = / (?:that|que)\b|(?: (?:the |that |le |que les )?((?:[A-Z]|national|laboratoires).{0,80}?)(?: to)?\b|)/
+      # @deprecated use Clauses::CONSIDERATIONS / Clauses::ACTIONS directly.
+      CONSIDERATIONS = Clauses::CONSIDERATIONS
+      # @deprecated use Clauses::ACTIONS directly.
+      ACTIONS = Clauses::ACTIONS
+      # @deprecated use Clauses::PREFIX directly.
+      PREFIX = Clauses::PREFIX
+      # @deprecated use Clauses::SUFFIX directly.
+      SUFFIX = Clauses::SUFFIX
 
       DOIREGEX = %r'\s+<p>\s+<b>DOI :</b> (.*?)\s+</p>\n\n'
 
@@ -95,16 +38,7 @@ module Bipm
         def replace_links(ps, res, lang)
           ps.css("a[href]").each do |a|
             href = a.attr("href")
-
-            href = href.gsub(%r'\Ahttps://www\.bipm\.org/', "")
-
-            # Correct links
-            href = href.gsub("/web/guest/", "/#{lang}/")
-
-            # Account for some mistakes from an upstream document
-            href = href.gsub(%r"\A/jen/", "/en/")
-            href = href.gsub(%r"\A/en/CGPM/jsp/", "/en/CGPM/db/")
-
+            href = Quirks.fix_href(href, language: lang)
             href = case href
               when %r'\A/(\w{2})/CGPM/db/(\d+)/(\d+)/(#.*)?\z',
                    %r'\A/jsp/(\w{2})/ViewCGPMResolution\.jsp\?CGPM=(\d+)&RES=(\d+)(#.*)?\z',
@@ -165,7 +99,7 @@ module Bipm
 
         def format_message(part)
           AsciiMath.asciidoc_extract_math(
-            Coradoc::Input::HTML.convert(part).strip.gsub("&nbsp;", " ").gsub(" \n", "\n")
+            Coradoc::Input::Html.convert(part).strip.gsub("&nbsp;", " ").gsub(" \n", "\n")
           )
         rescue
           warn "Bug in Coradoc, couldn't parse the following document:"
@@ -195,7 +129,7 @@ module Bipm
 
           refs = ng.css(".publication-card_reference a")
 
-          if rec_type.end_with? "?"
+          if rec_type&.end_with?("?")
             rec_type = case supertitle
               when /\AD[eé]claration/
                 "statement"
@@ -242,8 +176,6 @@ module Bipm
 
           ps = ng.css("div.journal-content-article").first
 
-          #binding.pry if ps.count != 1
-
           # Replace links
           Common.replace_links(ps, res, lang)
 
@@ -251,7 +183,6 @@ module Bipm
           Common.replace_centers(ps)
 
           doc = Common.ng_to_string(ps)
-          # doc = AsciiMath.html_to_asciimath(doc)
 
           if doc.match? DOIREGEX
             doc = doc.sub(DOIREGEX, "")
@@ -284,8 +215,8 @@ module Bipm
               next
             end
 
-            CONSIDERATIONS.any? do |k, v|
-              if parse =~ /\A#{PREFIX}#{k}\b/i
+            Clauses::CONSIDERATIONS.any? do |k, v|
+              if parse =~ /\A#{Clauses::PREFIX}#{k}\b/i
                 r["considerations"] << prev = {
                   "type" => v,
                   "date_effective" => date.to_s,
@@ -294,8 +225,8 @@ module Bipm
               end
             end && next
 
-            ACTIONS.any? do |k, v|
-              if parse =~ /\A#{PREFIX}#{k}\b/i
+            Clauses::ACTIONS.any? do |k, v|
+              if parse =~ /\A#{Clauses::PREFIX}#{k}\b/i
                 r["actions"] << prev = {
                   "type" => v,
                   "date_effective" => date.to_s,
@@ -321,17 +252,17 @@ module Bipm
             next if parse =~ /\A(|\[Cliquer ici\]|Click here|\.\.\.)\z/
 
             r["x-unparsed"] ||= []
-            r["x-unparsed"] << parse #ReverseAdoc.convert(part).strip
+            r["x-unparsed"] << parse
           end
 
           %w[considerations actions].each do |type|
-            map = type == "actions" ? ACTIONS : CONSIDERATIONS
+            map = type == "actions" ? Clauses::ACTIONS : Clauses::CONSIDERATIONS
             r[type] = r[type].map do |i|
               islist = false
 
               kk = nil
 
-              if map.any? { |k, v| (i["message"].split("\n").first =~ /\A\s*([*_]?)(#{PREFIX}#{k})\1?(#{SUFFIX})\1?\s*\z/i) && (kk = k) }
+              if map.any? { |k, v| (i["message"].split("\n").first =~ /\A\s*([*_]?)(#{Clauses::PREFIX}#{k})\1?(#{Clauses::SUFFIX})\1?\s*\z/i) && (kk = k) }
                 prefix = $2
                 suffix = $3
                 subject = $4
@@ -340,7 +271,7 @@ module Bipm
                 listitems = []
                 if (i["message"].split(/(?<!\+)\n(?!\+)/).all? { |j|
                   case j
-                  when /\A\s*[*_]?#{PREFIX}#{kk}/i
+                  when /\A\s*[*_]?#{Clauses::PREFIX}#{kk}/i
                     true
                   when /\A\s*\z/
                     true
@@ -357,7 +288,6 @@ module Bipm
               end
 
               if subject
-                #p subject
                 r["subject"] ||= []
                 r["subject"] << subject
               end
@@ -400,26 +330,7 @@ module Bipm
         end
 
         def extract_date(date_str)
-          return nil unless date_str
-
-          date = date_str.strip
-                         .gsub(/\s+/, " ")
-                         .gsub("février", "february") # 3 first letters must match English
-                         .gsub("juin", "june")
-                         .gsub("avril", "april")
-                         .gsub("mai", "may")
-                         .gsub("juillet", "july")
-                         .gsub(/ao[uû]t/, "august")
-                         .gsub("décembre", "december")
-                         .split(/, | to | au /) # Get last date
-                         .last
-          date = Date.parse(date)
-
-          binding.pry if date <= Date.parse("0000-01-01")
-
-          date
-        rescue Date::Error
-          binding.pry
+          Text::French.parse_date(date_str)
         end
 
         extend self

data/lib/bipm/data/importer/fetcher.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "mechanize"
+
+module Bipm
+  module Data
+    module Importer
+      # Orchestrates a single body's scrape. Picks the strategy declared
+      # by the Body, configures VCR/WebMock for the strategy's recording
+      # mode, then delegates the actual work. The CLI never touches VCR.
+      class Fetcher
+        attr_reader :body, :languages, :base_dir
+
+        def initialize(body:, languages: Language.all, base_dir: "data")
+          @body = body.is_a?(Body) ? body : Bodies.find(body)
+          @languages = Array(languages).map { |l| l.is_a?(Language) ? l : Language.find(l) }
+          @base_dir = base_dir
+        end
+
+        def call(agent: Mechanize.new)
+          strategy = strategy_class.new(body: body, base_dir: base_dir, languages: languages)
+          with_recording_mode(strategy.recording_mode) do
+            strategy.call(agent: agent)
+          end
+        end
+
+        private
+
+        def strategy_class
+          case body.strategy
+          when :static_index then Strategies::StaticIndex
+          when :spa_meetings then Strategies::SpaMeetings
+          else raise ArgumentError, "unknown strategy #{body.strategy.inspect} for body #{body.id.inspect}"
+          end
+        end
+
+        def with_recording_mode(mode)
+          return yield if mode == :cassette
+
+          disable_vcr_and_webmock!
+          yield
+        end
+
+        def disable_vcr_and_webmock!
+          begin
+            require "vcr"
+            VCR.eject_cassette
+            VCR.turn_off!(ignore_cassettes: true)
+          rescue LoadError, VCR::Errors::Error
+            # VCR not loaded or already turned off.
+          end
+          begin
+            require "webmock"
+            WebMock.disable!
+          rescue LoadError
+            # webmock not loaded.
+          end
+        end
+      end
+    end
+  end
+end