relaton-bipm 1.14.1 → 1.14.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 516c29a4d2e4ce02f6e2a1800f248f26ffa9b15c848d604ba5c100e05104acc8
-  data.tar.gz: 32455c13d2414e272d3ae9d9fe9b948116987439a81133fe235370eedac98fdc
+  metadata.gz: 34d720b316dbd942e2c5d630d2ae0f07b74331e4ef07f68715e84304bad0fb13
+  data.tar.gz: 38d36e34b998db6e4fa9e9f1a6e5306fadacea45b9a878cd14629eaaca2ef50d
 SHA512:
-  metadata.gz: dab2ecf54b507fad29bedd65870525f8c063d2299e7df03fd731f5382e43ecbd900931860b58c8c346ed98c3f1214242f97f795c549f20da9d0a65bb321191e3
-  data.tar.gz: eba8ab69d65e03ee43cfb3bd8529eb56c6836244897f71deba1799ebf33c513a375bb1fb4527920d721b11bd9cd6604d253a831d9d62c5703cd480d39ed9d8b4
+  metadata.gz: a22261617d5c3de8aad7ed410091331698630f958332c5feb0b215b9fafe9167015530e9e6ecb71046307a6775aa7a2a0a71dd4c12e4f982cb0bd259e021a267
+  data.tar.gz: 376bb090dd4d273b8039357d78280c9bc4f1555918920a55b22ec72cb8be87c4ce0a8469254ffc000256fa95069a271b65b978dc767d735acd4b3e141c5dea24

data/.gitignore

@@ -12,3 +12,4 @@
 .rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
 Gemfile.lock
 .vscode/
+rawdata-bipm-metrologia

data/Gemfile

@@ -3,5 +3,11 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in relaton_bipm.gemspec
 gemspec
 
+gem "byebug"
+gem "pry-byebug"
 gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
+gem "ruby-jing"
+gem "simplecov"
+gem "vcr"
+gem "webmock"

data/README.adoc

@@ -70,22 +70,35 @@ Allowed document names are:
 
 ==== Reference structure for Metrologia documents
 
-`BIPM Metrologia {JOURNAL} {VOLUME} {ISSUE} {PAGE}`
+`BIPM Metrologia {JOURNAL} {VOLUME} {ISSUE}`
 
-- `{JOURNAL}` - number of journal, required
-- `{VOLUME}` - number of volume, optional
-- `{ISSUE}` - number of issue, optional
-- `{PAGE}` - number of page, optional
+- `{JOURNAL}` - journal number, required
+- `{VOLUME}` - volume number, optional
+- `{ISSUE}` - issue number, optional
 
 ==== Reference structures for CCTF (CCDS), CGPM, CIPM documents
 
 ===== Basic pattern
 
 ----
-Long: {group name} -- {type} {number} ({year})
-Short: {group name} -- {type-abbrev} {number} ({year}, {lang})
+Long:
+{group name} -- {type} {number} ({year})
+{group name} {type} {number} ({year})
+{group name} {type} {year}-{zero_leading_number}
+
+Short:
+{group name} -- {type-abbrev} {number} ({year}, {lang})
+{group name} {type-abbrev} {number} ({year}, {lang})
 ----
 
+`group name` - a name of the group, required. A full list of group names is available https://github.com/metanorma/bipm-editor-guides/blob/main/sources/bipm-outcomes-en.adoc#appendix-a-bipm-groups-and-codes[here].
+`type` - a type of document, required. A list of types is: Resolution (Résolution), Recommendation (Recommandation), Decision (Décision), Meeting (Réunion), Declaration (Déclaration).
+`type-abbrev` - an abbreviation of the type, required. A list of abbreviations: RES (Resolution), REC (Recommendation), DECN (Decision).
+`number` - a number of the document, optional. Can be with part, e.g. `1-2`.
+`zero_leading_number` - a number of the document with a leading zero, required. Can be used when a document has a 1 or 2 digits number. It's `00` for documents without a number.
+`year` - a year of the document, optional.
+`lang` - a language of the document, optional. Can be `EN` or `FR`.
+
 ===== Special case pattern
 
 The basic pattern works fine for all, except for these 2 cases:
@@ -189,9 +202,9 @@ item = RelatonBipm::BipmBibliography.get "BIPM SI Brochure"
 ...
 
 # get BIPM Metrologia page
-bib = RelatonBipm::BipmBibliography.get "BIPM Metrologia 29 6 373"
-[relaton-bipm] ("BIPM Metrologia 29 6 373") fetching...
-[relaton-bipm] ("BIPM Metrologia 29 6 373") found Metrologia 29 6 373
+bib = RelatonBipm::BipmBibliography.get "BIPM Metrologia 29 6 001"
+[relaton-bipm] ("BIPM Metrologia 29 6 001") fetching...
+[relaton-bipm] ("BIPM Metrologia 29 6 001") found Metrologia 29 6 001
 => #<RelatonBipm::BipmBibliographicItem:0x007f8857f94d40
 ...
 
@@ -295,7 +308,7 @@ bib.link
  #<RelatonBib::TypedUri:0x00007fa6d6a29250 @content=#<Addressable::URI:0xc2b0 URI:https://doi.org/10.1088/0026-1394/29/6/001>, @type="doi">]
 ----
 
-=== Create bibliographic item from XML
+=== Create a bibliographic item from XML
 
 [source,ruby]
 ----
@@ -304,7 +317,7 @@ RelatonBipm::XMLParser.from_xml File.read('spec/fixtures/bipm_item.xml')
 ...
 ----
 
-=== Create bibliographic item from YAML
+=== Create a bibliographic item from YAML
 [source,ruby]
 ----
 hash = YAML.load_file 'spec/fixtures/bipm_item.yml'
@@ -321,6 +334,7 @@ RelatonBipm::BipmBibliographicItem.from_hash hash
 This gem uses the following datasets as data sources:
 - `bipm-data-outcomes` - looking for a local directory with the repository https://github.com/metanorma/bipm-data-outcomes
 - `bipm-si-brochute` - looking for a local directory with the repository https://github.com/metanorma/bipm-si-brochure
+- `rawdata-bipm-metrologia` - looking for a local directory with the repository https://github.com/relaton/rawdata-bipm-metrologia
 
 The method `RelatonBipm::DataFetcher.fetch(source, output: "data", format: "yaml")` fetches all the documents from the dataset and saves them to the `./data` folder in YAML format.
 Arguments:
@@ -342,6 +356,12 @@ Started at: 2022-06-23 09:37:12 +0200
 Stopped at: 2022-06-23 09:37:12 +0200
 Done in: 0 sec.
 => nil
+
+RelatonBipm::DataFetcher.fetch "rawdata-bipm-metrologia"
+Started at: 2022-06-23 09:39:12 +0200
+Stopped at: 2022-06-23 09:40:34 +0200
+Done in: 82 sec.
+=> nil
 ----
 
 == Development

data/grammars/basicdoc.rng

@@ -522,7 +522,6 @@
       <value>tip</value>
       <value>important</value>
       <value>caution</value>
-      <value>statement</value>
     </choice>
   </define>
   <define name="figure">

data/grammars/biblio.rng

@@ -216,6 +216,9 @@
       <optional>
         <ref name="fullname"/>
       </optional>
+      <zeroOrMore>
+        <ref name="credential"/>
+      </zeroOrMore>
       <zeroOrMore>
         <ref name="affiliation"/>
       </zeroOrMore>
@@ -232,6 +235,11 @@
       <ref name="FullNameType"/>
     </element>
   </define>
+  <define name="credential">
+    <element name="credential">
+      <text/>
+    </element>
+  </define>
   <define name="FullNameType">
     <choice>
       <group>
@@ -305,7 +313,9 @@
       <zeroOrMore>
         <ref name="affiliationdescription"/>
       </zeroOrMore>
-      <ref name="organization"/>
+      <optional>
+        <ref name="organization"/>
+      </optional>
     </element>
   </define>
   <define name="affiliationname">
@@ -1316,7 +1326,7 @@
       <value>commentaryOf</value>
       <value>hasCommentary</value>
       <value>related</value>
-      <value>complements</value>
+      <value>hasComplement</value>
       <value>complementOf</value>
       <value>obsoletes</value>
       <value>obsoletedBy</value>

data/lib/relaton_bipm/bipm_bibliography.rb

@@ -3,14 +3,6 @@ require "mechanize"
 module RelatonBipm
   class BipmBibliography
     GH_ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-bipm/master/".freeze
-    IOP_DOMAIN = "https://iopscience.iop.org".freeze
-    TRANSLATIONS = {
-      "Déclaration" => "Declaration",
-      "Réunion" => "Meeting",
-      "Recommandation" => "Recommendation",
-      "Résolution" => "Resolution",
-      "Décision" => "Decision",
-    }.freeze
 
     class << self
       # @param text [String]
@@ -18,14 +10,13 @@ module RelatonBipm
       def search(text, _year = nil, _opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         warn "[relaton-bipm] (\"#{text}\") fetching..."
         ref = text.sub(/^BIPM\s/, "")
-        item = ref.match?(/^Metrologia/i) ? get_metrologia(ref, magent) : get_bipm(ref, magent)
+        item = get_bipm(ref, magent)
         unless item
           warn "[relaton-bipm] (\"#{text}\") not found."
           return
         end
 
         warn("[relaton-bipm] (\"#{text}\") found #{item.docidentifier[0].id}")
-        item.fetched = Date.today.to_s
         item
       rescue Mechanize::ResponseCodeError => e
         raise RelatonBib::RequestError, e.message unless e.response_code == "404"
@@ -48,295 +39,28 @@ module RelatonBipm
         a
       end
 
-      # @param ref [String]
+      # @param reference [String]
       # @param agent [Mechanize]
       # @return [RelatonBipm::BipmBibliographicItem]
-      def get_bipm(ref, agent) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        # rf = ref.sub(/(?:(\d{1,2})\s)?\(?(\d{4})(?!-)\)?/) do
-        #   "#{$2}-#{$1.to_s.rjust(2, '0')}"
-        # end
-        ref.sub!("CCDS", "CCTF")
-        # TRANSLATIONS.each { |fr, en| rf.sub! fr, en }
-        path = Index.new.search ref
-        return unless path
+      def get_bipm(reference, agent) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        ref_id = Id.new reference
+        index = Relaton::Index.find_or_create :BIPM, url: "#{GH_ENDPOINT}index2.zip"
+        rows = index.search { |r| ref_id == r[:id] }
+        return unless rows.any?
 
-        url = "#{GH_ENDPOINT}#{path}"
+        url = "#{GH_ENDPOINT}#{rows.first[:file]}"
         resp = agent.get url
-        check_response resp
         return unless resp.code == "200"
 
         yaml = RelatonBib.parse_yaml resp.body, [Date]
-        # yaml["fetched"] = Date.today.to_s
+        yaml["fetched"] = Date.today.to_s
         bib_hash = HashConverter.hash_to_bib yaml
         BipmBibliographicItem.new(**bib_hash)
       end
 
-      # @param ref [String]
-      # @param agent [Mechanize]
-      # @return [RelatonBipm::BipmBibliographicItem]
-      def get_metrologia(ref, agent)
-        agent.redirect_ok = false
-        ref_arr = ref.split
-        case ref_arr.size
-        when 1 then get_journal agent
-        when 2 then get_volume ref_arr[1], agent
-        when 3 then get_issue(*ref_arr[1..2], agent)
-        when 4 then get_article_from_issue(*ref_arr[1..3], agent)
-        end
-      end
-
-      # @param agent [Mechanize]
-      # @return [RelatonBipm::BipmBibliographicItem]
-      def get_journal(agent)
-        url = "#{IOP_DOMAIN}/journal/0026-1394"
-        rsp = agent.get url
-        check_response rsp
-        rel = rsp.xpath('//select[@id="allVolumesSelector"]/option').map do |v|
-          { type: "partOf", bibitem: journal_rel(v) }
-        end
-        did = doc_id []
-        bibitem(formattedref: fref(did.id), docid: [did], link: blink(url), relation: rel)
-      end
-
-      # @param elm [Nokogiri::XML::Element]
-      def journal_rel(elm)
-        vol = elm[:value].split("/").last
-        did = doc_id [vol]
-        url = IOP_DOMAIN + elm[:value]
-        BipmBibliographicItem.new(formattedref: fref(did.id), docid: [did], link: blink(url))
-      end
-
-      # @param vol [String]
-      # @param agent [Mechanize]
-      # @return [RelatonBipm::BipmBibliographicItem]
-      def get_volume(vol, agent)
-        url = "#{IOP_DOMAIN}/volume/0026-1394/#{vol}"
-        rsp = agent.get url
-        check_response rsp
-        rel = rsp.xpath('//li[@itemprop="hasPart"]').map do |i|
-          { type: "partOf", bibitem: volume_rel(i, vol) }
-        end
-        did = doc_id [vol]
-        bibitem(formattedref: fref(did.id), docid: [did], link: blink(url), date: bdate(rsp), relation: rel,
-                extent: btextent(vol), series: series)
-      end
-
-      def volume_rel(elm, vol) # rubocop:disable Metrics/AbcSize
-        a = elm.at 'a[@itemprop="issueNumber"]'
-        ish = a[:href].split("/").last
-        url = IOP_DOMAIN + a[:href]
-        docid = doc_id [vol, ish]
-        t = elm.at "p"
-        title_fref = t ? { title: titles(t.text) } : { formattedref: fref(docid.id) }
-        BipmBibliographicItem.new(**title_fref, docid: [docid], link: blink(url))
-      end
-
-      # @param title [String]
-      # @return [RelatonBib::TypedTitleStringCollection]
-      def titles(title)
-        RelatonBib::TypedTitleString.from_string title, "en", "Latn", "text/html"
-      end
-
-      # @param vol [String]
-      # @param ish [String]
-      # @param agent [Mechanize]
-      # @return [RelatonBipm::BipmBibliographicItem]
-      def get_issue(vol, ish, agent) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        url = issue_url vol, ish
-        rsp = agent.get url
-        check_response rsp
-        rel = rsp.xpath('//div[@class="art-list-item-body"]').map do |a|
-          { type: "partOf", bibitem: issue_rel(a, vol, ish) }
-        end
-        did = doc_id [vol, ish]
-        title_fref = { title: issue_title(rsp) }
-        title_fref[:formattedref] = fref did.id unless title_fref[:title].any?
-        bibitem(**title_fref, link: blink(url), relation: rel, docid: [did],
-                              date: bdate(rsp), extent: btextent(vol, ish), series: series)
-      end
-
-      # @param ref [String]
-      # @return [RelatonBib::FormattedRef]
-      def fref(ref)
-        RelatonBib::FormattedRef.new content: ref, language: "en", script: "Latn"
-      end
-
-      # @param rsp [Mechanize::Page]
-      # @return [RelatonBib::TypedTitleStringCollection]
-      def issue_title(rsp)
-        t = rsp.at('//div[@id="wd-jnl-issue-title"]/h4')
-        return RelatonBib::TypedTitleStringCollection.new [] unless t
-
-        titles(t.text)
-      end
-
-      # @oaran vol [String]
-      # @param ish [String]
-      # @return [String]
-      def issue_url(vol, ish)
-        "#{IOP_DOMAIN}/issue/0026-1394/#{vol}/#{ish}"
-      end
-
-      # @param elm [Nokogiri::XML::Element]
-      # @param vol [String]
-      # @param ish [String]
-      # @return [RelatonBipm::BipmBibliographicItem]
-      def issue_rel(elm, vol, ish)
-        art = elm.at('div[@class="indexer"]').text
-        ref = elm.at('div/a[@class="art-list-item-title"]')
-        title = titles ref.text.strip
-        docid = doc_id [vol, ish, art]
-        link = blink IOP_DOMAIN + ref[:href]
-        BipmBibliographicItem.new(title: title, docid: [docid], link: link)
-      end
-
-      # @param content [RelatonBib::TypedTitleString]
-      # @return [RelatonBib::TypedTitleString]
-      def btitle(content)
-        RelatonBib::TypedTitleString.new type: "main", content: content, language: "en", script: "Latn"
-      end
-
-      # @param url [String]
-      # @return [String]
-      def blink(url)
-        [RelatonBib::TypedUri.new(type: "src", content: url)]
-      end
-
-      # @param rsp [Mechanize::Page]
-      # @return [Array<RelatonBib::BibliographicDate>]
-      def bdate(rsp)
-        date = rsp.at('//p[@itemprop="issueNumber"]|//h2[@itemprop="volumeNumber"]').text.split(", ").last
-        on = date.match?(/^\d{4}$/) ? date : Date.parse(date).strftime("%Y-%m")
-        [RelatonBib::BibliographicDate.new(type: "published", on: on)]
-      end
-
-      # @param args [Array<String>]
-      # @return [RelatonBib::DocumentIdentifier]
-      def doc_id(args)
-        id = args.clone.unshift "Metrologia"
-        RelatonBib::DocumentIdentifier.new(type: "BIPM", id: id.join(" "), primary: true)
-      end
-
-      # @param vol [String]
-      # @param ish [String]
-      # @param art [String]
-      # @param agent [Mechanize]
-      # @return [RelatonBipm::BipmBibliographicItem]
-      def get_article_from_issue(vol, ish, art, agent) # rubocop:disable Metrics/MethodLength
-        url = issue_url vol, ish
-        rsp = agent.get url
-        check_response rsp
-        link = rsp.at("//div[@class='indexer'][.='#{art}']/../div/a")
-        unless link
-          arts = rsp.xpath("//div[@class='indexer']").map(&:text)
-          warn "[relaton-bipm] No article is available at the specified start page \"#{art}\" in issue \"BIPM Metrologia #{vol} #{ish}\"."
-          warn "[relaton-bipm] Available articles in the issue start at the following pages: (#{arts.join(', ')})"
-          return
-        end
-
-        get_article link[:href], vol, ish, agent
-      end
-
-      # @param path [String]
-      # @param vol [String]
-      # @param ish [String]
-      # @param agent [Mechanize]
-      # @return [RelatonBipm::BipmBibliographicItem]
-      def get_article(path, vol, ish, agent) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-        agent.agent.allowed_error_codes = [403]
-        rsp = agent.get path
-        check_response rsp
-        title = rsp.at("//h1[@itemprop='headline']").children.to_xml
-        url = rsp.uri
-        bib = rsp.link_with(text: "BibTeX").href
-        rsp = agent.get bib
-        check_response rsp
-        bt = BibTeX.parse(rsp.body).first
-        bibitem(
-          docid: btdocid(bt), title: titles(title), date: btdate(bt),
-          abstract: btabstract(bt), doctype: bt.type.to_s, series: series,
-          link: btlink(bt, url), contributor: btcontrib(bt),
-          extent: btextent(vol, ish, bt.pages.to_s)
-        )
-      end
-
-      # @param args [Hash]
-      # @return [RelatonBipm::BipmBibliographicItem]
-      def bibitem(**args)
-        BipmBibliographicItem.new(
-          type: "article", language: ["en"], script: ["Latn"], **args,
-        )
-      end
-
-      # @return [Array<RelatonBib::Series>]
-      def series
-        [RelatonBib::Series.new(title: btitle("Metrologia"))]
-      end
-
-      # @param bibtex [BibTeX::Entry]
-      # @return [Array<RelatonBib::DocumentIdentifier>]
-      def btdocid(bibtex)
-        id = "#{bibtex.journal} #{bibtex.volume} #{bibtex.number} #{bibtex.pages.match(/^\d+/)}"
-        [
-          RelatonBib::DocumentIdentifier.new(type: "BIPM", id: id, primary: true),
-          RelatonBib::DocumentIdentifier.new(type: "DOI", id: bibtex.doi),
-        ]
-      end
-
-      # @param bibtex [BibTeX::Entry]
-      # @return [Array<RelatonBib::FormattedString>]
-      def btabstract(bibtex)
-        [RelatonBib::FormattedString.new(content: bibtex.abstract.to_s, language: "en", script: "Latn")]
-      end
-
-      # @param bibtex [BibTeX::Entry]
-      # @param ref [URI]
-      # @return [Array<RelatonBib::TypedUri>]
-      def btlink(bibtex, ref)
-        [
-          RelatonBib::TypedUri.new(type: "src", content: ref.to_s),
-          RelatonBib::TypedUri.new(type: "doi", content: bibtex.url.to_s),
-        ]
-      end
-
-      # @param bibtex [BibTeX::Entry]
-      # @return [Array<RelatonBib::BibliographicDate>]
-      def btdate(bibtex)
-        on = Date.new(bibtex.year.to_i, bibtex.month_numeric)
-        [RelatonBib::BibliographicDate.new(type: "published", on: on)]
-      end
-
-      # @param bibtex [BibTeX::Entry]
-      # @return [Array<Hash>]
-      def btcontrib(bibtex) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-        contribs = []
-        if bibtex.publisher && !bibtex.publisher.empty?
-          org = RelatonBib::Organization.new name: bibtex.publisher.to_s
-          contribs << { entity: org, role: [{ type: "publisher" }] }
-        end
-        return contribs unless bibtex.author && !bibtex.author.empty?
-
-        bibtex.author.split(" and ").inject(contribs) do |mem, name|
-          cname = RelatonBib::LocalizedString.new name, "en", "Latn"
-          name = RelatonBib::FullName.new completename: cname
-          author = RelatonBib::Person.new name: name
-          mem << { entity: author, role: [{ type: "author" }] }
-        end
-      end
-
-      #
-      # @param vol [String] volume
-      # @param ish [String] issue
-      # @param pgs [String] pages
-      #
-      # @return [Array<RelatonBib::BibItemLocality>]
-      #
-      def btextent(vol, ish = nil, pgs = nil)
-        ext = [RelatonBib::Locality.new("volume", vol)]
-        ext << RelatonBib::Locality.new("issue", ish) if ish
-        ext << RelatonBib::Locality.new("page", *pgs.split("--")) if pgs
-        ext
-      end
+      # def match_item(ids, ref_id)
+      #   ids.find { |id| Id.new(id) == ref_id }
+      # end
 
       # @param ref [String] the BIPM standard Code to look up (e..g "BIPM B-11")
       # @param year [String] not used
@@ -345,28 +69,6 @@ module RelatonBipm
       def get(ref, year = nil, opts = {})
         search(ref, year, opts)
       end
-
-      private
-
-      #
-      # Check HTTP response. Warn and rise error if response is not 200
-      #   or redirect to CAPTCHA.
-      #
-      # @param [Mechanize] rsp response
-      #
-      # @raise [RelatonBib::RequestError] if response is not 200
-      #
-      def check_response(rsp) # rubocop:disable Metrics/AbcSize
-        if rsp.code == "302"
-          warn "[relaton-bipm] This source employs anti-DDoS measures that unfortunately affects automated requests."
-          warn "[relaton-bipm] Please visit this link in your browser to resolve the CAPTCHA, then retry: #{rsp.uri}"
-          # warn "[relaton-bipm] #{rsp.uri} is redirected to #{rsp.header['location']}"
-          raise RelatonBib::RequestError, "cannot access #{rsp.uri}"
-        elsif rsp.code != "200" && rsp.code != "403"
-          warn "[read_bipm] can't acces #{rsp.uri} #{rsp.code}"
-          raise RelatonBib::RequestError, "cannot acces #{rsp.uri} #{rsp.code}"
-        end
-      end
     end
   end
 end

data/lib/relaton_bipm/bipm_si_brochure_parser.rb

@@ -6,7 +6,7 @@ module RelatonBipm
     # @param [RelatonBipm::DataFetcher] data_fetcher data fetcher
     #
     def initialize(data_fetcher)
-      @data_fetcher = data_fetcher
+      @data_fetcher = WeakRef.new data_fetcher
     end
 
     #
@@ -27,14 +27,18 @@ module RelatonBipm
       # puts "Ls #{Dir['bipm-si-brochure/*']}"
       # puts "Ls #{Dir['bipm-si-brochure/site/*']}"
       # puts "Ls #{Dir['bipm-si-brochure/site/documents/*']}"
-      Dir["bipm-si-brochure/site/documents/*.rxl"].each do |f|
+      Dir["bipm-si-brochure/_site/documents/*.rxl"].each do |f|
         puts "Parsing #{f}"
         docstd = Nokogiri::XML File.read f
         doc = docstd.at "/bibdata"
         hash1 = RelatonBipm::XMLParser.from_xml(doc.to_xml).to_hash
         fix_si_brochure_id hash1
-        outfile = File.join @data_fetcher.output, File.basename(f).sub(/(?:-(?:en|fr))?\.rxl$/, ".yaml")
-        @data_fetcher.index[[hash1["docnumber"] || File.basename(outfile, ".yaml")]] = outfile
+        basename = File.join @data_fetcher.output, File.basename(f).sub(/(?:-(?:en|fr))?\.rxl$/, "")
+        outfile = "#{basename}.#{@data_fetcher.ext}"
+        key = hash1["docnumber"] || basename
+        @data_fetcher.index[[key]] = outfile
+        @data_fetcher.index_new.add_or_update [key], outfile
+        @data_fetcher.index2.add_or_update Id.new(key).normalized_hash, outfile
         hash = if File.exist? outfile
                  warn_duplicate = false
                  hash2 = YAML.load_file outfile

data/lib/relaton_bipm/comment_periond.rb

@@ -15,7 +15,7 @@ module RelatonBipm
 
     # @param builder [Nokogiri::XML::builder]
     def to_xml(builder)
-      builder.send "comment-period" do
+      builder.send :"comment-period" do
         builder.from from.to_s
         builder.to to.to_s if to
       end

data/lib/relaton_bipm/data_fetcher.rb

@@ -1,6 +1,6 @@
 module RelatonBipm
   class DataFetcher
-    attr_reader :output, :format, :ext, :files, :index
+    attr_reader :output, :format, :ext, :files, :index, :index_new, :index2
 
     #
     # Initialize fetcher
@@ -15,6 +15,8 @@ module RelatonBipm
       @files = []
       @index_path = "index.yaml"
       @index = File.exist?(@index_path) ? YAML.load_file(@index_path) : {}
+      @index_new = Relaton::Index.find_or_create :BIPM, file: "index-bipm.yaml"
+      @index2 = Relaton::Index.find_or_create :BIPM, file: "index2.yaml"
     end
 
     #
@@ -43,8 +45,11 @@ module RelatonBipm
       case source
       when "bipm-data-outcomes" then DataOutcomesParser.parse(self)
       when "bipm-si-brochure" then BipmSiBrochureParser.parse(self)
+      when "rawdata-bipm-metrologia" then RawdataBipmMetrologia::Fetcher.fetch(self)
       end
-      File.write @index_path, @index.to_yaml, encoding: "UTF-8"
+      File.write @index_path, index.to_yaml, encoding: "UTF-8"
+      index_new.save
+      index2.save
     end
 
     #
@@ -54,15 +59,22 @@ module RelatonBipm
     # @param [RelatonBipm::BipmBibliographicItem] item document to save
     # @param [Boolean, nil] warn_duplicate Warn if document already exists
     #
-    # @return [<Type>] <description>
-    #
     def write_file(path, item, warn_duplicate: true)
+      content = serialize item
       if @files.include?(path)
         warn "File #{path} already exists" if warn_duplicate
       else
         @files << path
       end
-      File.write path, item.to_hash.to_yaml, encoding: "UTF-8"
+      File.write path, content, encoding: "UTF-8"
+    end
+
+    def serialize(item)
+      case @format
+      when "xml" then item.to_xml bibdata: true
+      when "yaml" then item.to_hash.to_yaml
+      when "bibxml" then item.to_bibxml
+      end
     end
   end
 end