relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/doi/processor.rb

@@ -0,0 +1,65 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Doi
+    class Processor < Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_doi
+        @prefix = "DOI"
+        @defaultprefix = %r{^doi:}
+        @idtype = "DOI"
+      end
+
+      # @param code [String] DOI
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [RelatonBib::BibliographicItem]
+      def get(code, _date, _opts)
+        require_relative "../doi"
+        Crossref.get(code)
+      end
+
+      #
+      # @param [String] source source name
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format
+      #
+      def fetch_data(_source, _opts)
+        Util.info "This processor does not support fetching data by source name. Use `get` method with DOI instead."
+      end
+
+      # @param xml [String]
+      # @return [Bib::ItemData]
+      def from_xml(xml)
+        require_relative "../doi"
+        Bib::Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Bib::ItemData]
+      def from_yaml(yaml)
+        require_relative "../doi"
+        Bib::Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../doi"
+        @grammar_hash ||= ::Relaton::Doi.grammar_hash
+      end
+
+      def remove_index_file
+        require_relative "../doi" # defines Util — needed when called cold
+        Util.info "This processor does not support index file. No action taken."
+      end
+
+      # Returns number of threads
+      # @return [Integer]
+      def threads = 2
+    end
+  end
+end

data/lib/relaton/doi/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Doi
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-doi".freeze
+    end
+  end
+end

data/lib/relaton/doi.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "relaton/bipm"
+require "relaton/ietf"
+require "relaton/ieee"
+require "relaton/nist"
+require_relative "version"
+require_relative "doi/util"
+require_relative "doi/parser"
+require_relative "doi/crossref"
+
+module Relaton
+  module Doi
+    extend self
+
+    def grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end

data/lib/relaton/ecma/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ecma
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/ecma/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ecma
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/ecma/bibliography.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal:true
+
+require "mechanize"
+
+module Relaton
+  module Ecma
+    # IETF bibliography module
+    module Bibliography
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-ecma/refs/heads/v2/"
+
+      class << self
+        #
+        # Search for a reference on the IETF website.
+        #
+        # @param ref [String] the ECMA standard reference to look up (e..g "ECMA-6")
+        #
+        # @return [Array<Hash>]
+        #
+        def search(ref)
+          refparts = parse_ref ref
+          return [] unless refparts
+
+          index = Relaton::Index.find_or_create :ECMA, url: "#{ENDPOINT}#{INDEXFILE}.zip", id_keys: %i[id ed vol]
+          index.search { |row| match_ref refparts, row }
+        end
+
+        def parse_ref(ref)
+          %r{^ECMA[-\s]
+            (?<id>(?:\d[\d-]*|\w+/\d+))
+            (?:\sed(?<ed>[\d.]+))?
+            (?:\svol(?<vol>\d+))?
+          }x.match ref
+        end
+
+        def match_ref(refparts, row) # rubocop:disable Metrics/AbcSize
+          row[:id][:id].match?(/^ECMA[-\s]#{refparts[:id]}/) &&
+            (refparts[:ed].nil? || row[:id][:ed] == refparts[:ed]) &&
+            (refparts[:vol].nil? || row[:id][:vol] == refparts[:vol])
+        end
+
+        # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
+        # @param year [String] not used
+        # @param opts [Hash] not used
+        # @return [Relaton::Ecma::ItemData] Relaton of reference
+        def get(code, _year = nil, _opts = {})
+          Util.info "Fetching from Relaton repository ...", key: code
+          result = fetch_doc(code)
+          if result
+            Util.info "Found: `#{result.docidentifier.first.content}`", key: code
+            # item
+          else
+            Util.info "Not found.", key: code
+          end
+          result
+        end
+
+        def compare_edition_volume(aaa, bbb)
+          comp = bbb[:id][:ed] <=> aaa[:id][:ed]
+          comp.zero? ? aaa[:id][:vol] <=> bbb[:id][:vol] : comp
+        end
+
+        def fetch_doc(code) # rubocop:disable Metrics/AbcSize
+          row = search(code).min { |a, b| compare_edition_volume a, b }
+          return unless row
+
+          url = "#{ENDPOINT}#{row[:file]}"
+          resp = Mechanize.new.get(url)
+          Item.from_yaml(resp.body).tap { |item| item.fetched = Date.today.to_s }
+        rescue Mechanize::ResponseCodeError => e
+          return if e.response_code == "404"
+
+          raise Relaton::RequestError, "No document found for #{code} reference. #{e.message}"
+        rescue Mechanize::RedirectLimitReachedError, Timeout::Error,
+            Mechanize::UnauthorizedError, Mechanize::UnsupportedSchemeError,
+            Mechanize::ResponseReadError, Mechanize::ChunkedTerminationError => e
+          raise Relaton::RequestError, "No document found for #{code} reference. #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ecma/data_fetcher.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "English"
+require "mechanize"
+require "relaton/core"
+require_relative "../ecma"
+require_relative "parser_common"
+require_relative "page_fetcher"
+require_relative "standard_parser"
+require_relative "memento_parser"
+require_relative "edition_parser"
+require_relative "data_parser"
+
+module Relaton
+  module Ecma
+    class DataFetcher < Core::DataFetcher
+      URL = "https://www.ecma-international.org/publications-and-standards/"
+      SOURCES = %w[standards technical-reports mementos].freeze
+
+      def index
+        @index ||= Relaton::Index.find_or_create :ecma, file: "#{INDEXFILE}.yaml"
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def agent
+        @agent ||= Mechanize.new.tap { |a| a.user_agent_alias = Mechanize::AGENT_ALIASES.keys.sample }
+      end
+
+      # @param bib [Relaton::Ecma::ItemData]
+      def write_file(bib)
+        file = filename bib
+        if @files.include? file
+          Util.warn "Duplicate file #{file}"
+        else
+          @files << file
+          File.write file, serialize(bib), encoding: "UTF-8"
+          index.add_or_update index_id(bib), file
+        end
+      end
+
+      def filename(bib)
+        id = bib.docidentifier[0].content
+        id += " #{bib.edition.content}" if bib.edition
+        locality = locality_with_volume bib
+        id += " #{locality.reference_from}" if locality
+        output_file id
+      end
+
+      def index_id(bib)
+        { id: bib.docidentifier[0].content }.tap do |i|
+          i[:ed] = bib.edition.content if bib.edition
+          locality = locality_with_volume bib
+          i[:vol] = locality.reference_from if locality
+        end
+      end
+
+      def locality_with_volume(bib)
+        bib.extent.each do |e|
+          locality = e.locality.find { |l| l.type == "volume" }
+          return locality if locality
+        end
+        nil
+      end
+
+      def to_xml(bib) = bib.to_xml(bibdata: true)
+      def to_yaml(bib) = bib.to_yaml
+      def to_bibxml(bib) = bib.to_rfcxml
+
+      # @param hit [Nokogiri::XML::Element]
+      def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        DataParser.new(hit, @errors).parse.each { |item| write_file item }
+      end
+
+      # @param type [String]
+      def html_index(type) # rubocop:disable Metrics/MethodLength
+        result = agent.get "#{URL}#{type}/"
+        # @last_call_time = Time.now
+        result.xpath(
+          "//li/span[1]/a",
+          "//div[contains(@class, 'entry-content-wrapper')][.//a[.='Download']]",
+        ).each do |hit|
+          parse_page(hit)
+        rescue StandardError => e
+          Util.error { "#{e.message}\n#{e.backtrace}" }
+        end
+      end
+
+      #
+      # Fetch data from Ecma website.
+      #
+      # @return [void]
+      #
+      def fetch(_ = nil)
+        SOURCES.each { |source| html_index source }
+        index.save
+        report_errors
+      end
+    end
+  end
+end

data/lib/relaton/ecma/data_parser.rb

@@ -0,0 +1,49 @@
+module Relaton
+  module Ecma
+    class DataParser
+      include ParserCommon
+
+      #
+      # Initialize parser
+      #
+      # @param [Nokogiri::XML::Element] hit document hit
+      # @param [Hash] errors error tracking hash
+      #
+      def initialize(hit, errors = {})
+        @hit = hit
+        @errors = errors
+      end
+
+      # @return [Array<Relaton::Ecma::ItemData>]
+      def parse
+        if @hit[:href]
+          parse_standard
+        else
+          parse_memento
+        end
+      end
+
+      private
+
+      def parse_standard
+        doc = PageFetcher.new.get(@hit[:href])
+        parser = StandardParser.new(hit: @hit, doc: doc, errors: @errors)
+        bib = parser.to_bib_hash
+        bib[:contributor] = contributor
+        items = [ItemData.new(**bib)]
+        edition_parser = EditionParser.new(
+          doc: doc, bib: bib, errors: @errors,
+          translation_source: parser.translation_source
+        )
+        items + edition_parser.parse
+      end
+
+      def parse_memento
+        parser = MementoParser.new(hit: @hit, errors: @errors)
+        bib = parser.to_bib_hash
+        bib[:contributor] = contributor
+        [ItemData.new(**bib)]
+      end
+    end
+  end
+end

data/lib/relaton/ecma/edition_parser.rb

@@ -0,0 +1,80 @@
+module Relaton
+  module Ecma
+    class EditionParser
+      include ParserCommon
+
+      # @param [Mechanize::Page] doc document page
+      # @param [Hash] bib base bibliographic item attributes
+      # @param [Hash] errors error tracking hash
+      # @param [Array] translation_source precomputed translation sources
+      def initialize(doc:, bib:, errors: {}, translation_source: [])
+        @doc = doc
+        @bib = bib
+        @errors = errors
+        @translation_source = translation_source
+      end
+
+      # @return [Array<Relaton::Ecma::ItemData>] editions
+      def parse # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+        return [] unless @doc
+
+        docid = @bib[:docidentifier]
+        @doc.xpath('//div[@id="main"]/div[1]/div/main/article/div/div/standard/div/ul/li').map do |hit|
+          bib = @bib.dup
+          id, ed, bib[:date], vol = edition_id_parts hit.at("./span", "./a").text
+          bib[:source] = edition_source(hit) + edition_translation_source(ed)
+          next if ed.nil? || ed.empty?
+
+          bib[:docidentifier] = id.nil? || id.empty? ? docid : fetch_docidentifier(id)
+          @errors[:edition_docidentifier] &&= bib[:docidentifier].empty?
+          bib[:edition] = Bib::Edition.new(content: ed)
+          bib[:extent] = create_extent(vol)
+          @errors[:edition_extent] &&= bib[:extent].nil?
+          ItemData.new(**bib)
+        end.compact
+      end
+
+      #
+      # Parse edition and date
+      #
+      # @param [String] text identifier text
+      #
+      # @return [Array<String,nil,Array<Relaton::Bib::Date>>] edition and date
+      #
+      def edition_id_parts(text) # rubocop:disable Metrics/MethodLength
+        %r{^
+          (?<id>\w+(?:[\d-]+|\sTR/\d+)),?\s
+          (?:Volume\s(?<vol>[\d.]+),?\s)?
+          (?<ed>[\d.]+)(?:st|nd|rd|th)?\sedition
+          (?:[,.]\s(?<dt>\w+\s\d+))?
+        }x =~ text
+        date = [dt].compact.map do |d|
+          on = Date.strptime(d, "%B %Y").strftime("%Y-%m")
+          Bib::Date.new(type: "published", at: on)
+        end
+        [id, ed, date, vol]
+      end
+
+      def edition_source(hit)
+        es = { "src" => hit.at("./a"), "pdf" => hit.at("./span/a") }.map do |type, a|
+          Bib::Uri.new(type: type, content: a[:href]) if a
+        end.compact
+        @errors[:edition_source] &&= es.empty?
+        es
+      end
+
+      def create_extent(vol)
+        return unless vol && !vol.empty?
+
+        locality = Bib::Locality.new(type: "volume", reference_from: vol)
+        [Bib::Extent.new(locality: [locality])]
+      end
+
+      private
+
+      def edition_translation_source(edition)
+        @translation_source.select { |s| s[:ed] == edition }.map { |s| s[:source] }
+      end
+    end
+  end
+end

data/lib/relaton/ecma/ext.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Ecma
+    class Ext < Bib::Ext
+      def get_schema_version = Relaton.schema_versions["relaton-model-ecma"]
+    end
+  end
+end

data/lib/relaton/ecma/item.rb

@@ -0,0 +1,11 @@
+require_relative "ext"
+
+module Relaton
+  module Ecma
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+    end
+  end
+end

data/lib/relaton/ecma/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Ecma
+    class ItemData < Relaton::Bib::ItemData
+    end
+  end
+end

data/lib/relaton/ecma/memento_parser.rb

@@ -0,0 +1,60 @@
+module Relaton
+  module Ecma
+    class MementoParser
+      include ParserCommon
+
+      ATTRS = %i[docidentifier title date source ext].freeze
+
+      # @param [Nokogiri::XML::Element] hit document hit
+      # @param [Hash] errors error tracking hash
+      def initialize(hit:, errors: {})
+        @hit = hit
+        @errors = errors
+      end
+
+      # @return [Hash] bibliographic item attributes
+      def to_bib_hash
+        bib = default_bib_hash
+        ATTRS.each { |a| bib[a] = send "fetch_#{a}" }
+        bib
+      end
+
+      private
+
+      # @return [Array<Relaton::Bib::Docidentifier>]
+      def fetch_docidentifier
+        code = "ECMA MEM/#{@hit.at('div[1]//p').text}"
+        docid = super(code)
+        @errors[:memento_docidentifier] &&= docid.empty?
+        docid
+      end
+
+      # @return [Array<Relaton::Bib::Title>]
+      def fetch_title
+        year = @hit.at("div[1]//p").text
+        content = "\"Memento #{year}\" for year #{year}"
+        result = [Bib::Title.new(content: content, language: "en", script: "Latn")]
+        @errors[:memento_title] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Date>]
+      def fetch_date
+        date = @hit.at("div[2]//p").text
+        on = Date.strptime(date, "%B %Y").strftime "%Y-%m"
+        result = [Bib::Date.new(type: "published", at: on)]
+        @errors[:memento_date] &&= result.empty?
+        result
+      end
+
+      # @return [Array<Relaton::Bib::Uri>]
+      def fetch_source
+        result = @hit.xpath("./div/section/div/p/a").map do |a|
+          Bib::Uri.new(type: "pdf", content: a[:href])
+        end
+        @errors[:memento_source] &&= result.empty?
+        result
+      end
+    end
+  end
+end

data/lib/relaton/ecma/page_fetcher.rb

@@ -0,0 +1,27 @@
+module Relaton
+  module Ecma
+    class PageFetcher
+      def initialize
+        @agent = Mechanize.new
+        @agent.user_agent_alias = Mechanize::AGENT_ALIASES.keys[rand(21)]
+      end
+
+      #
+      # Get page with retries
+      #
+      # @param [String] url url to fetch
+      #
+      # @return [Mechanize::Page] document
+      #
+      def get(url)
+        3.times do |n|
+          sleep n
+          doc = @agent.get url
+          return doc
+        rescue StandardError => e
+          Util.error e.message
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ecma/parser_common.rb

@@ -0,0 +1,33 @@
+module Relaton
+  module Ecma
+    module ParserCommon
+      def default_bib_hash
+        {
+          type: "standard", language: ["en"], script: ["Latn"], place: [Bib::Place.new(city: "Geneva")]
+        }
+      end
+
+      def contributor
+        orgname = Bib::TypedLocalizedString.new(content: "Ecma International", language: "en", script: "Latn")
+        org = Bib::Organization.new name: [orgname]
+        role = Bib::Contributor::Role.new type: "publisher"
+        [Bib::Contributor.new(organization: org, role: [role])]
+      end
+
+      # @return [Array<Relaton::Bib::Docidentifier>]
+      def fetch_docidentifier(id = nil)
+        return [] if id.nil? || id.empty?
+
+        [Bib::Docidentifier.new(type: "ECMA", content: id, primary: true)]
+      end
+
+      def fetch_ext
+        Ext.new(doctype: fetch_doctype, flavor: "ecma")
+      end
+
+      def fetch_doctype
+        Bib::Doctype.new content: "document"
+      end
+    end
+  end
+end

data/lib/relaton/ecma/processor.rb

@@ -0,0 +1,66 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Ecma
+    class Processor < Relaton::Core::Processor
+      def initialize
+        @short = :relaton_ecma
+        @prefix = "ECMA"
+        @defaultprefix = /^ECMA(-|\s)/
+        @idtype = "ECMA"
+        @datasets = %w[ecma-standards]
+      end
+
+      # @param code [String]
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [Relaton::Ecma::ItemData, nil]
+      def get(code, date, opts)
+        require_relative "../ecma"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Fetch all the documents from a source
+      #
+      # @param [String] source source name (iec-harmonized-all, iec-harmonized-latest)
+      # @param [Hash] opts
+      # @option opts [String] :output directory to output documents
+      # @option opts [String] :format output format (xml, yaml, bibxml)
+      #
+      def fetch_data(source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(source, **opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Ecma::ItemData]
+      def from_xml(xml)
+        require_relative "../ecma"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Ecma::ItemData]
+      def from_yaml(yaml)
+        require_relative "../ecma" # defines Item — cache reads hit this cold
+        Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../ecma"
+        @grammar_hash ||= Ecma.grammar_hash
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../ecma"
+        Relaton::Index.find_or_create(:ECMA, url: true, file: "#{INDEXFILE}.yaml").remove_file
+      end
+    end
+  end
+end