relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/nist/mods_parser.rb

@@ -0,0 +1,302 @@
+# frozen_string_literal: true
+
+require "pubid"
+require "loc_mods"
+
+module Relaton
+  module Nist
+    class ModsParser
+      RELATION_TYPES = {
+        "otherVersion" => "editionOf",
+        "preceding" => "updates",
+        "succeeding" => "updatedBy",
+      }.freeze
+
+      ATTRS = %i[type docidentifier title source abstract date contributor
+                  relation place series].freeze
+
+      def initialize(doc, series, errors = {})
+        @doc = doc
+        @series = series
+        @errors = errors
+      end
+
+      # @return [Bib::ItemData]
+      def parse
+        args = ATTRS.each_with_object({}) do |attr, hash|
+          hash[attr] = send("parse_#{attr}")
+        end
+        args[:ext] = Ext.new(doctype: parse_doctype, flavor: "nist")
+        ItemData.new(**args)
+      end
+
+      def parse_type = "standard"
+
+      # @return [Array<Bib::Docidentifier>]
+      def parse_docidentifier
+        ids = [
+          { type: "NIST", content: pub_id, primary: true },
+          { type: "DOI", content: parse_doi },
+        ].reject { |id| id[:content].nil? || id[:content].empty? }
+        @errors[:docidentifier] &&= ids.empty?
+        # All ids share the Nist::Docidentifier collection type (Item#docidentifier)
+        # so YAML serialization stays valid; the DOI is not canonical NIST pubid
+        # form, so the class keeps it as the raw string (see Docidentifier#content=).
+        ids.map { |id| Docidentifier.new(**id) }
+      end
+
+      # @return [String]
+      def pub_id = get_id_from_str parse_doi
+
+      def get_id_from_str(str)
+        return if str.nil? || str.empty?
+
+        # DOIs arrive dotted ("NIST.HB.135e2022-upd1"); that MR-style form
+        # parses through a path that skips UpdateCodes date lookup and drops
+        # letter suffixes. Space the leading "PUB.SERIES." prefix so it parses
+        # like the canonical short form (only the separator dots are touched,
+        # not any inside the number). Then force :human rendering.
+        spaced = str.sub(/\A(NIST|NBS)\.([A-Z][A-Za-z]*)\./, '\1 \2 ')
+        ::Pubid::Nist::Identifier.parse(spaced).to_s(format: :human)
+      rescue ::Pubid::Errors::ParseError
+        str.gsub(".", " ").sub(/^[\D]+/, &:upcase)
+      end
+
+      # @return [String]
+      def replace_wrong_doi(id)
+        case id
+        when "NBS.CIRC.sup" then "NBS.CIRC.24e7sup"
+        when "NBS.CIRC.supJun1925-Jun1926" then "NBS.CIRC.24e7sup2"
+        when "NBS.CIRC.supJun1925-Jun1927" then "NBS.CIRC.24e7sup3"
+        when "NBS.CIRC.24supJuly1922" then "NBS.CIRC.24e6sup"
+        when "NBS.CIRC.24supJan1924" then "NBS.CIRC.24e6sup2"
+        else id
+        end
+      end
+
+      def parse_doi
+        url = @doc.location.reduce(nil) { |m, l| m || l.url.detect { |u| u.usage == "primary display" } }
+        return if url.nil?
+
+        id = remove_doi_prefix(url.content)
+        return if id.nil?
+
+        replace_wrong_doi(id)
+      end
+
+      def remove_doi_prefix(id) = id.match(/10\.6028\/(.+)/)&.send(:[], 1)
+
+      # @return [Array<Bib::Title>]
+      def parse_title
+        title = @doc.title_info.reduce([]) do |a, ti|
+          next a if ti.type == "alternative"
+
+          a += ti.title.map { |t| create_title(t, "title-main", ti.non_sort&.first) }
+          next a unless ti.sub_title
+
+          a + ti.sub_title.map { |t| create_title(t, "title-part") }
+        end
+        if title.size > 1
+          content = title.map { |t| t.content }.join(" - ")
+          title << create_title(content, "main")
+        elsif title.size == 1
+          title[0].instance_variable_set :@type, "main"
+        end
+        @errors[:title] &&= title.empty?
+        title
+      end
+
+      def create_title(title, type, non_sort = nil)
+        content = title.gsub("\n", " ").squeeze(" ").strip
+        content = "#{non_sort.content}#{content}".squeeze(" ") if non_sort
+        Bib::Title.new content: content, type: type, language: "en", script: "Latn"
+      end
+
+      def parse_source
+        source = @doc.location.map do |location|
+          url = location.url.first
+          type = url.usage == "primary display" ? "doi" : "src"
+          Bib::Uri.new content: url.content, type: type
+        end
+        @errors[:source] &&= source.empty?
+        source
+      end
+
+      def parse_abstract
+        abstract = Array(@doc.abstract).map do |a|
+          content = a.content.gsub("\n", " ").squeeze(" ").strip
+          Bib::Abstract.new content: content, language: "en",
+                                           script: "Latn"
+        end
+        @errors[:abstract] &&= abstract.empty?
+        abstract
+      end
+
+      def parse_date
+        date = @doc.origin_info[0].date_issued.map do |di|
+          create_date(di, "issued")
+        end.compact
+        @errors[:date] &&= date.empty?
+        date
+      end
+
+      def create_date(date, type)
+        Date.new type: type, at: decode_date(date)
+      rescue ::Date::Error
+      end
+
+      def decode_date(date)
+        if date.encoding == "marc" && date.content.size == 6
+          ::Date.strptime(date.content, "%y%m%d").to_s
+        elsif date.encoding == "iso8601"
+          ::Date.strptime(date.content, "%Y%m%d").to_s
+        else date.content
+        end
+      end
+
+      def parse_doctype = Doctype.new(content: "standard")
+
+      def parse_contributor
+        seen = []
+        contributor = @doc.name.filter_map do |name|
+          entity, default_role = create_entity(name)
+          next unless entity
+
+          # A name may appear twice — as the `usage="primary"` main entry and
+          # again as a plain added entry. Keep the first (primary) occurrence
+          # and drop the duplicate, so the lead author is retained, not doubled.
+          key = contributor_key(name)
+          next if seen.include?(key)
+
+          seen << key
+          role = (name.role || []).reduce([]) do |a, r|
+            a + r.role_term.map { |rt| Bib::Contributor::Role.new(type: rt.content) }
+          end
+          role << Bib::Contributor::Role.new(type: default_role) if role.empty?
+          create_contributor(entity, role)
+        end
+        @errors[:contributor] &&= contributor.empty?
+        contributor
+      end
+
+      # De-duplication signature for a contributor: its type, its non-date name
+      # parts, and its authority id — so identically-named people with distinct
+      # authority ids are still kept separate.
+      def contributor_key(name)
+        parts = name.name_part.reject(&:type).map { |p| p.content&.strip }
+        [name.type, parts, name.name_identifier&.first&.content]
+      end
+
+      def create_contributor(entity, role)
+        case entity
+        when Bib::Person
+          Bib::Contributor.new(role: role, person: entity)
+        when Bib::Organization
+          Bib::Contributor.new(role: role, organization: entity)
+        end
+      end
+
+      def create_entity(name)
+        case name.type
+        when "personal" then [create_person(name), "author"]
+        when "corporate" then [create_org(name), "publisher"]
+        end
+      end
+
+      def create_person(name)
+        # exclude typed name parts because they are not actual name parts
+        cname = name.name_part.reject(&:type).map(&:content).join(" ")
+        completename = Bib::LocalizedString.new(content: cname, language: "en")
+        fname = Bib::FullName.new(completename: completename)
+        name_id = name.name_identifier&.first
+        identifier = []
+        if name_id
+          identifier << Bib::Person::Identifier.new(type: "uri",
+                                                     content: name_id.content)
+        end
+        Bib::Person.new(name: fname, identifier: identifier)
+      end
+
+      def create_org(name)
+        names = name.name_part.reject(&:type).map do |n|
+          Bib::TypedLocalizedString.new(
+            content: n.content.gsub("\n", " ").squeeze(" ").strip,
+          )
+        end
+        url = name.name_identifier&.first&.content
+        identifier = []
+        if url
+          identifier << Bib::OrganizationType::Identifier.new(type: "uri",
+                                                               content: url)
+        end
+        Bib::Organization.new(name: names, identifier: identifier)
+      end
+
+      def parse_relation
+        relations = Array(@doc.related_item).reject { |ri| ri.type == "series" }.filter_map do |ri|
+          type = RELATION_TYPES[ri.type]
+          bibitem = create_related_item(ri)
+          Relation.new(type: type, bibitem: bibitem) if bibitem
+        end
+        @errors[:relation] &&= relations.empty?
+        relations
+      end
+
+      def create_related_item(item)
+        item_id = get_id_from_str related_item_id(item)
+        return if item_id.nil? || item_id.empty?
+
+        docid = Docidentifier.new(type: "NIST", content: item_id)
+        fref = Bib::Formattedref.new(content: item_id)
+        ItemData.new(docidentifier: [docid], formattedref: fref)
+      end
+
+      # Derive the related document's DOI id. Prefer an `otherType` that carries
+      # a `10.6028` DOI — it may be embedded in prose (e.g. "Translation of |a
+      # 10.6028/NIST.SP.1299" or "Supersedes … https://doi.org/10.6028/…"), not
+      # only a bare DOI — otherwise fall back to the `<name><namePart>`. Returns
+      # nil (relation skipped by #create_related_item) when neither yields a DOI,
+      # so a related item with no name and no DOI never aborts the crawl.
+      def related_item_id(item)
+        id = if item.other_type&.include?("10.6028")
+               item.other_type
+             else
+               item.name&.first&.name_part&.first&.content
+             end
+        return if id.nil?
+
+        doi = remove_doi_prefix(id)
+        return if doi.nil?
+
+        replace_wrong_doi(doi)
+      end
+
+      def parse_place
+        place = @doc.origin_info.select { |p| p.event_type == "publisher" }.map do |p|
+          pl = p.place[0].place_term[0].content
+          /(?<city>\w+), (?<state>\w+)/ =~ pl
+          Bib::Place.new(city: city, region: create_region(state))
+        end
+        @errors[:place] &&= place.empty?
+        place
+      end
+
+      def create_region(state)
+        [Bib::Place::RegionType.new(iso: state)]
+      rescue ArgumentError
+        []
+      end
+
+      def parse_series
+        series = Array(@doc.related_item).select { |ri| ri.type == "series" }.map do |ri|
+          tinfo = ri.title_info[0]
+          tcontent = tinfo.title[0].strip
+          title = Bib::Title.new(content: tcontent)
+          Bib::Series.new(title: [title], number: tinfo.part_number&.first)
+        end
+        @errors[:series] &&= series.empty?
+        series
+      end
+    end
+  end
+end

data/lib/relaton/nist/processor.rb

@@ -0,0 +1,69 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Nist
+    class Processor < Core::Processor
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_nist
+        @prefix = "NIST"
+        @pubid_flavor = :Nist # global prefixes sourced from Pubid::Nist.prefixes
+        @defaultprefix = %r{^(NIST|NBS|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC|FIPS)(/[^\s])?\s}
+        @idtype = "NIST"
+        @datasets = %w[nist-tech-pubs]
+      end
+
+      # @param code [String]
+      # @param date [String, NilClass] year
+      # @param opts [Hash]
+      # @return [Relaton::Nist::ItemData]
+      def get(code, date = nil, opts = {})
+        require_relative "../nist"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Fetch all the documents from a source
+      #
+      # @param [String] _source source name
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format
+      #
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Nist::ItemData]
+      def from_xml(xml)
+        require_relative "../nist"
+        Item.from_xml(xml)
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Nist::ItemData]
+      def from_yaml(yaml)
+        require_relative "../nist"
+        Item.from_yaml(yaml)
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../nist"
+        @grammar_hash ||= Nist.grammar_hash
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../nist"
+        Relaton::Index.find_or_create(
+          :nist, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/nist/pubs_export.rb

@@ -0,0 +1,69 @@
+require "singleton"
+
+module Relaton
+  module Nist
+    class PubsExport
+      include Singleton
+
+      DOMAIN = "https://csrc.nist.gov".freeze
+      PUBS_EXPORT = URI.join(DOMAIN, "/CSRC/media/feeds/metanorma/pubs-export")
+      DATAFILEDIR = File.expand_path ".relaton/nist", Dir.home
+      DATAFILE = File.expand_path "pubs-export.zip", DATAFILEDIR
+
+      def initialize
+        @mutex = Mutex.new
+      end
+
+      #
+      # Fetches json data form server
+      #
+      # @return [Array<Hash>] json data
+      #
+      def data
+        @mutex.synchronize do
+          @data ||= begin
+            ctime = File.ctime DATAFILE if File.exist? DATAFILE
+            if !ctime || ctime.to_date < ::Date.today || File.size(DATAFILE).zero?
+              fetch_data(ctime)
+            end
+            unzip
+          end
+        end
+      end
+
+      private
+
+      #
+      # Fetch data form server and save it to file
+      #
+      # @param ctime [Time, nil] file creation time
+      #
+      def fetch_data(ctime)
+        if !ctime || ctime < last_modified
+          @data = nil
+          uri_open = URI.method(:open) || Kernel.method(:open)
+          FileUtils.mkdir_p DATAFILEDIR
+          IO.copy_stream(uri_open.call("#{PUBS_EXPORT}.zip"), DATAFILE)
+        end
+      end
+
+      def last_modified
+        uri = URI("#{PUBS_EXPORT}.meta")
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+          Time.httpdate(http.head(uri.request_uri)["last-modified"])
+        end
+      end
+
+      #
+      # upack zip file
+      #
+      # @return [Array<Hash>] json data
+      #
+      def unzip
+        Zip::File.open(DATAFILE) do |zf|
+          JSON.parse zf.first.get_input_stream.read
+        end
+      end
+    end
+  end
+end

data/lib/relaton/nist/relation.rb

@@ -0,0 +1,10 @@
+require_relative "item_base"
+
+module Relaton
+  module Nist
+    class Relation
+      attribute :type, :string, values: %w[obsoletedBy supersedes supersededBy]
+      attribute :bibitem, ItemBase
+    end
+  end
+end