relaton-bipm 1.14.1 → 1.14.3

data/lib/relaton_bipm/data_outcomes_parser.rb

@@ -7,13 +7,21 @@ module RelatonBipm
       "Statement" => "DECL",
     }.freeze
 
+    TRANSLATIONS = {
+      "Déclaration" => "Declaration",
+      "Réunion" => "Meeting",
+      "Recommandation" => "Recommendation",
+      "Résolution" => "Resolution",
+      "Décision" => "Decision",
+    }.freeze
+
     #
     # Create data-outcomes parser
     #
     # @param [RelatonBipm::DataFetcher] data_fetcher data fetcher
     #
     def initialize(data_fetcher)
-      @data_fetcher = data_fetcher
+      @data_fetcher = WeakRef.new data_fetcher
     end
 
     #
@@ -68,21 +76,11 @@ module RelatonBipm
     # @param [String] dir output directory
     #
     def fetch_meeting(en_file, body, type, dir) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      en = RelatonBib.parse_yaml File.read(en_file, encoding: "UTF-8"), [Date]
-      en_md = en["metadata"]
-      fr_file = en_file.sub "en", "fr"
-      fr = RelatonBib.parse_yaml File.read(fr_file, encoding: "UTF-8"), [Date]
-      fr_md = fr["metadata"]
-      gh_src = "https://raw.githubusercontent.com/metanorma/bipm-data-outcomes/"
-      src_en = gh_src + en_file.split("/")[-3..].unshift("main").join("/")
-      src_fr = gh_src + fr_file.split("/")[-3..].unshift("main").join("/")
-      src = [
-        { type: "src", content: src_en, language: "en", script: "Latn" },
-        { type: "src", content: src_fr, language: "fr", script: "Latn" },
-      ]
+      _, en, fr_file, fr = read_files en_file
+      en_md, fr_md, num, part = meeting_md en, fr
+      src = meeting_links en_file, fr_file
 
-      /^(?<num>\d+)(?:-_(?<part>\d+))?-\d{4}$/ =~ en_md["url"].split("/").last
-      file = "#{num}.yaml"
+      file = "#{num}.#{@data_fetcher.ext}"
       path = File.join dir, file
       hash = bibitem body: body, type: type, en: en_md, fr: fr_md, num: num, src: src, pdf: en["pdf"]
       if @data_fetcher.files.include?(path) && part
@@ -92,13 +90,13 @@ module RelatonBipm
         has_part_item = RelatonBipm::BipmBibliographicItem.from_hash(yaml)
         has_part_item.relation << RelatonBib::DocumentRelation.new(type: "partOf", bibitem: item)
         @data_fetcher.write_file path, has_part_item, warn_duplicate: false
-        path = File.join dir, "#{num}-#{part}.yaml"
+        path = File.join dir, "#{num}-#{part}.#{@data_fetcher.ext}"
       elsif part
         hash[:title].each { |t| t[:content] = t[:content].sub(/\s\(.+\)$/, "") }
         h = bibitem body: body, type: type, en: en_md, fr: fr_md, num: num, src: src, pdf: en["pdf"]
         add_part h, part
         part_item = RelatonBipm::BipmBibliographicItem.new(**h)
-        part_item_path = File.join dir, "#{num}-#{part}.yaml"
+        part_item_path = File.join dir, "#{num}-#{part}.#{@data_fetcher.ext}"
         @data_fetcher.write_file part_item_path, part_item
         add_to_index part_item, part_item_path
         hash[:relation] = [RelatonBib::DocumentRelation.new(type: "partOf", bibitem: part_item)]
@@ -111,6 +109,40 @@ module RelatonBipm
       fetch_resolution body: body, en: en, fr: fr, dir: dir, src: src, num: num
     end
 
+    #
+    # Read English and French files
+    #
+    # @param [String] en_file Path to English file
+    #
+    # @return [Array<Hash, String, nil>] English / French metadata and file path
+    #
+    def read_files(en_file)
+      fr_file = en_file.sub "en", "fr"
+      [en_file, fr_file].map do |file|
+        if File.exist? file
+          data = RelatonBib.parse_yaml(File.read(file, encoding: "UTF-8"), [Date])
+          path = file
+        end
+        [path, data]
+      end.flatten
+    end
+
+    def meeting_md(eng, frn)
+      en_md = eng["metadata"]
+      num, part = en_md["identifier"].to_s.split("-")
+      [en_md, frn&.dig("metadata"), num, part]
+    end
+
+    def meeting_links(en_file, fr_file)
+      gh_src = "https://raw.githubusercontent.com/metanorma/bipm-data-outcomes/"
+      { "en" => en_file, "fr" => fr_file }.map do |lang, file|
+        next unless file
+
+        src = gh_src + file.split("/")[-3..].unshift("main").join("/")
+        { type: "src", content: src, language: lang, script: "Latn" }
+      end.compact
+    end
+
     #
     # Parse BIPM resolutions and write them to YAML files
     #
@@ -146,9 +178,7 @@ module RelatonBipm
         hash[:contributor] = contributors date, args[:body]
         hash[:structuredidentifier] = RelatonBipm::StructuredIdentifier.new docnumber: num
         item = RelatonBipm::BipmBibliographicItem.new(**hash)
-        file = year
-        file += "-#{num_justed}" # if num.size < 4
-        file += ".yaml"
+        file = "#{year}-#{num_justed}.#{@data_fetcher.ext}"
         out_dir = File.join args[:dir], r["type"].downcase
         FileUtils.mkdir_p out_dir
         path = File.join out_dir, file
@@ -209,6 +239,9 @@ module RelatonBipm
       end
       key << item.docidentifier.detect { |i| i.language == "fr" }.id
       @data_fetcher.index[key] = path
+      @data_fetcher.index_new.add_or_update key, path
+      key2 = Id.new(item.docnumber).normalized_hash
+      @data_fetcher.index2.add_or_update key2, path
     end
 
     #
@@ -344,8 +377,7 @@ module RelatonBipm
       docnum = create_docnum args[:body], args[:type], args[:num], args[:en]["date"]
       hash = { title: [], type: "proceedings", doctype: args[:type],
                place: [RelatonBib::Place.new(city: "Paris")] }
-      hash[:title] << create_title(args[:en]["title"], "en") if args[:en]["title"]
-      hash[:title] << create_title(args[:fr]["title"], "fr") if args[:fr]["title"]
+      hash[:title] = create_titles args.slice(:en, :fr)
       hash[:date] = [{ type: "published", on: args[:en]["date"] }]
       hash[:docid] = create_docids docnum
       hash[:docnumber] = docnum # .sub(" --", "").sub(/\s\(\d{4}\)/, "")
@@ -358,6 +390,12 @@ module RelatonBipm
       hash
     end
 
+    def create_titles(data)
+      data.each_with_object([]) do |(lang, md), mem|
+        mem << create_title(md["title"], lang.to_s) if md && md["title"]
+      end
+    end
+
     #
     # Create links
     #
@@ -366,12 +404,13 @@ module RelatonBipm
     # @return [Array<Hash>] Array of links
     #
     def create_links(**args)
-      links = [
-        { type: "citation", content: args[:en]["url"], language: "en", script: "Latn" },
-        { type: "citation", content: args[:fr]["url"], language: "fr", script: "Latn" },
-      ]
+      links = args.slice(:en, :fr).each_with_object([]) do |(lang, md), mem|
+        next unless md && md["url"]
+
+        mem << { type: "citation", content: md["url"], language: lang.to_s, script: "Latn" }
+      end
       RelatonBib.array(args[:pdf]).each { |pdf| links << { type: "pdf", content: pdf } }
-      links += args[:src] if args[:src]&.any?
+      links += args[:src] if args[:src]
       links
     end
 
@@ -456,8 +495,8 @@ module RelatonBipm
     # @return [RelatonBib::DocumentIdentifier] french document ID
     #
     def create_docid_fr(en_id)
-      tr = BipmBibliography::TRANSLATIONS.detect { |_, v| en_id.include? v }
-      id = en_id.sub tr[1], tr[0]
+      tr = TRANSLATIONS.detect { |_, v| en_id.include? v }
+      id = tr ? en_id.sub(tr[1], tr[0]) : en_id
       make_docid(id: id, type: "BIPM", primary: true, language: "fr", script: "Latn")
     end
 

data/lib/relaton_bipm/id_parser.rb

@@ -0,0 +1,134 @@
+module RelatonBipm
+  class Id
+    class Parser < Parslet::Parser
+      rule(:space) { match("\s").repeat(1) }
+      rule(:space?) { space.maybe }
+      rule(:comma) { str(",") >> space? }
+      rule(:lparen) { str("(") }
+      rule(:rparen) { str(")") }
+      rule(:slash) { str("/") }
+
+      rule(:delimeter) { str("--") >> space }
+      rule(:delimeter?) { delimeter.maybe }
+
+      rule(:lang) { comma >> match["A-Z"].repeat(2, 2).as(:lang) }
+      rule(:lang?) { lang.maybe }
+
+      rule(:number) { match["0-9-"].repeat(1).as(:number) >> space? }
+      rule(:number?) { number.maybe }
+
+      rule(:year) { match["0-9"].repeat(4, 4).as(:year) }
+      rule(:year_paren) { lparen >> year >> lang? >> rparen }
+      rule(:num_year) { number? >> year_paren }
+      rule(:year_num) { year >> str("-") >> number }
+      rule(:num_and_year) { num_year | year_num | number }
+
+      rule(:sect) { lparen >> match["IVX"].repeat >> rparen }
+      rule(:suff) { match["a-zA-Z-"].repeat(1) }
+      rule(:cgmp) { str("CGPM") }
+      rule(:cipm) { str("CIPM") >> (str(" MRA") | match["A-Z-"]).maybe }
+      rule(:cc) { str("CC") >> suff >> sect.maybe }
+      rule(:jc) { str("JC") >> suff }
+      rule(:cec) { str("CEC") }
+      rule(:wgms) { str("WG-MS") }
+      rule(:group) { (cgmp | cipm | cc | jc | cec | wgms).as(:group) }
+
+      rule(:type) { match["[:alpha:]"].repeat(1).as(:type) >> space }
+
+      rule(:type_group) { type >> group >> slash >> num_and_year }
+      rule(:group_type) { group >> space >> delimeter? >> type >> num_and_year }
+      rule(:outcome) { group_type | type_group }
+
+      rule(:append) { comma >> str("Appendix") >> space >> number }
+      rule(:brochure) { str("SI").as(:group) >> space >> str("Brochure").as(:type) >> append.maybe }
+
+      rule(:metrologia) { str("Metrologia").as(:group) >> (space >> match["a-zA-Z0-9\s"].repeat(1).as(:number)).maybe }
+
+      rule(:result) { outcome | brochure | metrologia }
+
+      root :result
+    end
+
+    TYPES = {
+      "Resolution" => "RES",
+      "Résolution" => "RES",
+      "Recommendation" => "REC",
+      "Recommandation" => "REC",
+      "Decision" => "DECN",
+      "Décision" => "DECN",
+      "Declaration" => "Déclaration",
+      "Réunion" => "Meeting",
+    }.freeze
+
+    # @return [Hash] the parsed id components
+    attr_accessor :id
+
+    #
+    # Create a new Id object
+    #
+    # @param [String] id id string
+    #
+    def initialize(id)
+      @id = Parser.new.parse(id)
+    rescue Parslet::ParseFailed => e
+      warn "[relaton-bipm] Incorrect reference: #{id}"
+      # warn "[relaton-bipm] #{e.parse_failure_cause.ascii_tree}"
+      raise RelatonBib::RequestError, e
+    end
+
+    #
+    # Compare two Id objects
+    #
+    # @param [RelatonBipm::Id, Hash] other the other Id object
+    #
+    # @return [Boolean] true if the two Id objects are equal
+    #
+    def ==(other)
+      other_hash = other.is_a?(Id) ? other.normalized_hash : other
+      hash = normalized_hash
+      hash.delete(:year) unless other_hash[:year]
+      other_hash.delete(:year) unless hash[:year]
+      hash.delete(:lang) unless other_hash[:lang]
+      other_hash.delete(:lang) unless hash[:lang]
+      hash == other_hash
+    end
+
+    #
+    # Transform ID parts.
+    # Traslate type into abbreviation, remove leading zeros from number
+    #
+    # @return [Hash] the normalized ID parts
+    #
+    def normalized_hash # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+      @normalized_hash ||= begin
+        hash = { group: id[:group].to_s.sub("CCDS", "CCTF") }
+        hash[:type] = normalized_type if id[:type]
+        norm_num = normalized_number
+        hash[:number] = norm_num unless norm_num.nil? || norm_num.empty?
+        hash[:year] = id[:year].to_s if id[:year]
+        hash[:lang] = id[:lang].to_s if id[:lang]
+        hash
+      end
+    end
+
+    #
+    # Translate type into abbreviation
+    #
+    # @return [String] the normalized type
+    #
+    def normalized_type
+      TYPES[id[:type].to_s] || id[:type].to_s
+    end
+
+    #
+    # Remove leading zeros from number
+    #
+    # @return [String, nil] the normalized number
+    #
+    def normalized_number
+      return unless id[:number]
+
+      id[:number].to_s.sub(/^0+/, "")
+    end
+  end
+end

data/lib/relaton_bipm/processor.rb

@@ -9,7 +9,7 @@ module RelatonBipm
       @prefix = "BIPM"
       @defaultprefix = %r{^(?:BIPM|CCTF|CCDS|CGPM|CIPM)(?!\w)}
       @idtype = "BIPM"
-      @datasets = %w[bipm-data-outcomes bipm-si-brochure]
+      @datasets = %w[bipm-data-outcomes bipm-si-brochure rawdata-bipm-metrologia]
     end
 
     # @param code [String]
@@ -21,10 +21,11 @@ module RelatonBipm
     end
 
     #
-    # Fetch all the documents from https://github.com/metanorma/bipm-data-outcomes
-    #   or https://github.com/metanorma/bipm-si-brochure
+    # Fetch all the documents from https://github.com/metanorma/bipm-data-outcomes,
+    #   https://github.com/metanorma/bipm-si-brochure, https://github.com/relaton/rawdata-bipm-metrologia
     #
-    # @param [String] source source name
+    # @param [String] source source name (bipm-data-outcomes, bipm-si-brochure,
+    #   rawdata-bipm-metrologia)
     # @param [Hash] opts
     # @option opts [String] :output directory to output documents
     # @option opts [String] :format

data/lib/relaton_bipm/rawdata_bipm_metrologia/article_parser.rb

@@ -0,0 +1,311 @@
+module RelatonBipm
+  module RawdataBipmMetrologia
+    class ArticleParser
+      ATTRS = %i[docid title contributor date copyright abstract relation series
+                 extent type doctype].freeze
+      #
+      # Create new parser and parse document
+      #
+      # @param [Nokogiri::XML::Element] doc document XML element
+      #
+      # @return [RelatonBipm::BipmBibliographicItem] document
+      #
+      def self.parse(doc)
+        new(doc).parse
+      end
+
+      #
+      # Initialize parser
+      #
+      # @param [Nokogiri::XML::Element] doc XML document
+      #
+      def initialize(doc)
+        @doc = doc
+        @meta = @doc.at("./front/article-meta")
+      end
+
+      #
+      # Create new document
+      #
+      # @return [RelatonBipm::BipmBibliographicItem] document
+      #
+      def parse
+        attrs = ATTRS.to_h { |a| [a, send("parse_#{a}")] }
+        BipmBibliographicItem.new(**attrs)
+      end
+
+      #
+      # Parse docid
+      #
+      # @return [Array<RelatonBib::DocumentIdentifier>] array of document identifiers
+      #
+      def parse_docid
+        pubid = "#{journal_title} #{volume_issue_article}"
+        primary_id = create_docid pubid, "BIPM", true
+        @meta.xpath("./article-id[@pub-id-type='doi']")
+          .each_with_object([primary_id]) do |id, m|
+          m << create_docid(id.text, id["pub-id-type"])
+        end
+      end
+
+      #
+      # Parse volume, issue and page
+      #
+      # @return [Array<String>] array of volume, issue and page
+      #
+      def volume_issue_article
+        volume = @meta.at("./volume").text
+        issue = @meta.at("./issue").text
+        # page = @doc.at("./front/article-meta/fpage")&.text || manuscript
+        [volume, issue, article].join(" ")
+      end
+
+      def article
+        @meta.at("./article-id[@pub-id-type='manuscript']").text.match(/[^_]+$/).to_s
+      end
+
+      #
+      # Parse journal title
+      #
+      # @return [String] journal title
+      #
+      def journal_title
+        @doc.at("./front/journal-meta/journal-title-group/journal-title").text
+      end
+
+      #
+      # Create document identifier
+      #
+      # @param [String] id document id
+      # @param [String] type id type
+      # @param [Boolean, nil] primary is primary id
+      #
+      # @return [RelatonBib::DocumentIdentifier] document identifier
+      #
+      def create_docid(id, type, primary = nil)
+        RelatonBib::DocumentIdentifier.new id: id, type: type, primary: primary
+      end
+
+      #
+      # Parse title
+      #
+      # @return [Array<RelatonBib::TypedTitleString>] array of title strings
+      #
+      def parse_title
+        @meta.xpath("./title-group/article-title").map do |t|
+          next if t.text.empty?
+
+          RelatonBib::TypedTitleString.new content: t.text, language: t[:"xml:lang"], script: "Latn"
+        end.compact
+      end
+
+      #
+      # Parse contributor
+      #
+      # @return [Array<RelatonBib::Contributor>] array of contributors
+      #
+      def parse_contributor
+        @meta.xpath("./contrib-group/contrib").map do |c|
+          entity = create_person(c) || create_organization(c)
+          RelatonBib::ContributionInfo.new(entity: entity, role: [type: c[:"contrib-type"]])
+        end
+      end
+
+      def create_person(contrib)
+        name = contrib.at("./name")
+        return unless name
+
+        RelatonBib::Person.new name: fullname(name), affiliation: affiliation(contrib)
+      end
+
+      def create_organization(contrib)
+        RelatonBib::Organization.new name: contrib.at("./collab").text
+      end
+
+      #
+      # Parse affiliations
+      #
+      # @param [Nokogiri::XML::Element] contrib contributor element
+      #
+      # @return [Array<RelatonBib::Affiliation>] array of affiliations
+      #
+      def affiliation(contrib) # rubocop:disable Metrics/AbcSize
+        contrib.xpath("./xref[@ref-type='aff']").map do |x|
+          a = @meta.at("./contrib-group/aff[@id='#{x[:rid]}']/label/following-sibling::node()")
+          parts = a.text.split(", ")
+          orgname = parts[0..-3].join(", ")
+          city, country = parts[-2..]
+          address = []
+          address << RelatonBib::Address.new(city: city, country: country) if city && country
+          org = RelatonBib::Organization.new name: orgname, contact: address
+          RelatonBib::Affiliation.new organization: org
+        end
+      end
+
+      #
+      # Create full name
+      #
+      # @param [Nokogiri::XML::Element] contrib contributor element
+      #
+      # @return [RelatonBib::FullName] full name
+      #
+      def fullname(name)
+        fname = forename name.at("./given-names")
+        sname = name.at("./surname").text
+        surname = RelatonBib::LocalizedString.new sname, "en", "Latn"
+        RelatonBib::FullName.new surname: surname, forename: fname
+      end
+
+      #
+      # Parse forename
+      #
+      # @param [String] given_name given name
+      #
+      # @return [Array<RelatonBib::Forename>] array of forenames
+      #
+      def forename(given_name) # rubocop:disable Metrics/MethodLength
+        return [] unless given_name
+
+        given_name.text.scan(/(\w+)(?:\s(\w)(?:\s|$))?/).map do |nm, int|
+          if nm.size == 1
+            name = nil
+            init = nm
+          else
+            name = nm
+            init = int
+          end
+          RelatonBib::Forename.new(content: name, language: ["en"], script: ["Latn"], initial: init)
+        end
+      end
+
+      #
+      # Parse date
+      #
+      # @return [Array<RelatonBib::BibliographicDate>] array of dates
+      #
+      def parse_date
+        on = dates.min
+        [RelatonBib::BibliographicDate.new(type: "published", on: on)]
+      end
+
+      #
+      # Parse date
+      #
+      # @yield [date, type] date and type
+      #
+      # @return [Array<String, Object>] string date or whatever block returns
+      #
+      def dates
+        @meta.xpath("./pub-date").map do |d|
+          month = date_part(d, "month")
+          day = date_part(d, "day")
+          date = "#{d.at('./year').text}-#{month}-#{day}"
+          block_given? ? yield(date, d[:"pub-type"]) : date
+        end
+      end
+
+      def date_part(date, type)
+        part = date.at("./#{type}")&.text
+        return "01" if part.nil? || part.empty?
+
+        part.rjust(2, "0")
+      end
+
+      #
+      # Parse copyright
+      #
+      # @return [Array<RelatonBib::CopyrightAssociation>] array of copyright associations
+      #
+      def parse_copyright
+        @meta.xpath("./permissions").each_with_object([]) do |l, m|
+          from = l.at("./copyright-year")
+          next unless from
+
+          owner = l.at("./copyright-statement").text.split(" & ").map do |c|
+            /(?<name>[A-z]+(?:\s[A-z]+)*)/ =~ c
+            org = RelatonBib::Organization.new name: name
+            RelatonBib::ContributionInfo.new(entity: org)
+          end
+          m << RelatonBib::CopyrightAssociation.new(owner: owner, from: from.text)
+        end
+      end
+
+      #
+      # Parse abstract
+      #
+      # @return [Array<RelatonBib::FormattedString>] array of abstracts
+      #
+      def parse_abstract
+        @meta.xpath("./abstract").map do |a|
+          RelatonBib::FormattedString.new(
+            content: a.inner_html, language: a[:"xml:lang"], script: ["Latn"], format: "text/html",
+          )
+        end
+      end
+
+      #
+      # Parese relation
+      #
+      # @return [Array<RelatonBib::DocumentRelation>] array of document relations
+      #
+      def parse_relation
+        dates do |d, t|
+          RelatonBib::DocumentRelation.new(type: "hasManifestation", bibitem: bibitem(d, t))
+        end
+      end
+
+      #
+      # Create bibitem
+      #
+      # @param [String] date
+      # @param [String] type date type
+      #
+      # @return [RelatonBipm::BipmBibliographicItem] bibitem
+      #
+      def bibitem(date, type)
+        dt = RelatonBib::BibliographicDate.new(type: type, on: date)
+        carrier = type == "epub" ? "online" : "print"
+        medium = RelatonBib::Medium.new carrier: carrier
+        BipmBibliographicItem.new title: parse_title, date: [dt], medium: medium
+      end
+
+      #
+      # Parse series
+      #
+      # @return [Array<RelatonBib::Series>] array of series
+      #
+      def parse_series
+        title = RelatonBib::TypedTitleString.new(
+          content: journal_title, language: ["en"], script: ["Latn"],
+        )
+        [RelatonBib::Series.new(title: title)]
+      end
+
+      #
+      # Parse extent
+      #
+      # @return [Array<RelatonBib::Extent>] array of extents
+      #
+      def parse_extent
+        @meta.xpath("./volume|./issue|./fpage").map do |e|
+          if e.name == "fpage"
+            type = "page"
+            to = @meta.at("./lpage")&.text
+          else
+            type = e.name
+          end
+          RelatonBib::Locality.new type, e.text, to
+        end
+        # %w[volume issue page].map.with_index do |t, i|
+        #   RelatonBib::Locality.new t, volume_issue_page[i]
+        # end
+      end
+
+      def parse_type
+        "article"
+      end
+
+      alias_method :parse_doctype, :parse_type
+    end
+  end
+end