relaton-iso 1.20.0 → 2.0.0.pre.alpha.1

data/bin/ruby-parse

@@ -1,27 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# This file was generated by Bundler.
-#
-# The application 'ruby-parse' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
-
-bundle_binstub = File.expand_path("bundle", __dir__)
-
-if File.file?(bundle_binstub)
-  if File.read(bundle_binstub, 300).include?("This file was generated by Bundler")
-    load(bundle_binstub)
-  else
-    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
-Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
-  end
-end
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("parser", "ruby-parse")

data/bin/ruby-rewrite

@@ -1,27 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# This file was generated by Bundler.
-#
-# The application 'ruby-rewrite' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
-
-bundle_binstub = File.expand_path("bundle", __dir__)
-
-if File.file?(bundle_binstub)
-  if File.read(bundle_binstub, 300).include?("This file was generated by Bundler")
-    load(bundle_binstub)
-  else
-    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
-Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
-  end
-end
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("parser", "ruby-rewrite")

data/bin/safe_yaml

@@ -1,29 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# This file was generated by Bundler.
-#
-# The application 'safe_yaml' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-require "pathname"
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
-  Pathname.new(__FILE__).realpath)
-
-bundle_binstub = File.expand_path("../bundle", __FILE__)
-
-if File.file?(bundle_binstub)
-  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
-    load(bundle_binstub)
-  else
-    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
-Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
-  end
-end
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("safe_yaml", "safe_yaml")

data/bin/thor

@@ -1,27 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# This file was generated by Bundler.
-#
-# The application 'thor' is installed as part of a gem, and
-# this file is here to facilitate running it.
-#
-
-ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
-
-bundle_binstub = File.expand_path("bundle", __dir__)
-
-if File.file?(bundle_binstub)
-  if File.read(bundle_binstub, 300).include?("This file was generated by Bundler")
-    load(bundle_binstub)
-  else
-    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
-Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
-  end
-end
-
-require "rubygems"
-require "bundler/setup"
-
-load Gem.bin_path("thor", "thor")

data/lib/relaton_iso/data_fetcher.rb

@@ -1,246 +0,0 @@
-module RelatonIso
-  # Fetch all the documents from ISO website.
-  class DataFetcher
-    #
-    # Initialize data fetcher.
-    #
-    # @param [String] output output directory
-    # @param [String] format format of output files (yaml, bibxml, xml)
-    #
-    def initialize(output, format) # rubocop:disable Metrics/AbcSize
-      @output = output
-      @format = format
-      @ext = format.sub(/^bib/, "")
-      @files = Set.new
-      @queue = ::Queue.new
-      @mutex = Mutex.new
-      @gh_issue = Relaton::Logger::Channels::GhIssue.new "relaton/relaton-iso", "Error fetching ISO documents"
-      Relaton.logger_pool[:gh_issue] = Relaton::Logger::Log.new(@gh_issue, levels: [:error])
-      @errors = Hash.new(true)
-    end
-
-    def index
-      @index ||= Relaton::Index.find_or_create :iso, file: HitCollection::INDEXFILE
-    end
-
-    def iso_queue
-      @iso_queue ||= RelatonIso::Queue.new
-    end
-
-    #
-    # Initialize data fetcher and fetch data.
-    #
-    # @param [String] output output directory (default: "data")
-    # @param [String] format format of output files. Allowed: yaml (default), bibxml, xml
-    #
-    # @return [void]
-    #
-    def self.fetch(output: "data", format: "yaml")
-      t1 = Time.now
-      Util.info "Started at: #{t1}"
-      FileUtils.mkdir_p output
-      new(output, format).fetch
-      t2 = Time.now
-      Util.info "Stopped at: #{t2}"
-      Util.info "Done in: #{(t2 - t1).round} sec."
-    end
-
-    #
-    # Go through all ICS and fetch all documents.
-    #
-    # @return [void]
-    #
-    def fetch # rubocop:disable Metrics/AbcSize
-      Util.info "Scrapping ICS pages..."
-      fetch_ics
-      Util.info "(#{Time.now}) Scrapping documents..."
-      fetch_docs
-      iso_queue.save
-      # index.sort! { |a, b| compare_docids a, b }
-      index.save
-      repot_errors
-    end
-
-    def repot_errors
-      @errors.select { |_, v| v }.each_key do |k|
-        Util.error "Failed to fetch #{k}"
-      end
-      @gh_issue.create_issue
-    end
-
-    #
-    # Fetch ICS page recursively and store all the links to documents in the iso_queue.
-    #
-    # @param [String] path path to ICS page
-    #
-    def fetch_ics
-      threads = Array.new(3) { thread { |path| fetch_ics_page(path) } }
-      fetch_ics_page "/standards-catalogue/browse-by-ics.html"
-      sleep(1) until @queue.empty?
-      threads.size.times { @queue << :END }
-      threads.each(&:join)
-    end
-
-    def fetch_ics_page(path)
-      resp = get_redirection path
-      unless resp
-        Util.error "Failed fetching ICS page #{url(path)}"
-        return
-      end
-
-      page = Nokogiri::HTML(resp.body)
-      parse_doc_links page
-      parse_ics_links page
-    end
-
-    def parse_doc_links(page)
-      doc_links = page.xpath "//td[@data-title='Standard and/or project']/div/div/a"
-      @errors[:doc_links] &&= doc_links.empty?
-      doc_links.each { |item| iso_queue.add_first item[:href].split("?").first }
-    end
-
-    def parse_ics_links(page)
-      ics_links = page.xpath("//td[@data-title='ICS']/a")
-      @errors[:ics_links] &&= ics_links.empty?
-      ics_links.each { |item| @queue << item[:href] }
-    end
-
-    def url(path)
-      Scrapper::DOMAIN + path
-    end
-
-    #
-    # Get the page from the given path. If the page is redirected, get the
-    # page from the new path.
-    #
-    # @param [String] path path to the page
-    #
-    # @return [Net::HTTPOK, nil] HTTP response
-    #
-    def get_redirection(path) # rubocop:disable Metrics/MethodLength
-      try = 0
-      uri = URI url(path)
-      begin
-        get_response uri
-      rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED => e
-        try += 1
-        retry if check_try try, uri
-
-        Util.warn "Failed fetching #{uri}, #{e.message}"
-      end
-    end
-
-    def get_response(uri)
-      resp = Net::HTTP.get_response(uri)
-      resp.code == "302" ? get_redirection(resp["location"]) : resp
-    end
-
-    def check_try(try, uri)
-      if try < 3
-        Util.warn "Timeout fetching #{uri}, retrying..."
-        sleep 1
-        true
-      end
-    end
-
-    def fetch_docs
-      threads = Array.new(3) { thread { |path| fetch_doc(path) } }
-      iso_queue[0..10_000].each { |docpath| @queue << docpath }
-      threads.size.times { @queue << :END }
-      threads.each(&:join)
-    end
-
-    #
-    # Fetch document from ISO website.
-    #
-    # @param [String] docpath document page path
-    #
-    # @return [void]
-    #
-    def fetch_doc(docpath)
-      doc = Scrapper.parse_page docpath, errors: @errors
-      @mutex.synchronize { save_doc doc, docpath }
-    rescue StandardError => e
-      Util.warn "Fail fetching document: #{url(docpath)}\n#{e.message}\n#{e.backtrace}"
-    end
-
-    # def compare_docids(id1, id2)
-    #   Pubid::Iso::Identifier.create(**id1).to_s <=> Pubid::Iso::Identifier.create(**id2).to_s
-    # end
-
-    #
-    # save document to file.
-    #
-    # @param [RelatonIsoBib::IsoBibliographicItem] doc document
-    #
-    # @return [void]
-    #
-    def save_doc(doc, docpath) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      docid = doc.docidentifier.detect(&:primary)
-      file_name = docid.id.gsub(/[\s\/:]+/, "-").downcase
-      file = File.join @output, "#{file_name}.#{@ext}"
-      if File.exist?(file)
-        rewrite_with_same_or_newer doc, docid, file, docpath
-      else
-        write_file file, doc, docid
-      end
-      iso_queue.move_last docpath
-    end
-
-    def rewrite_with_same_or_newer(doc, docid, file, docpath)
-      hash = YAML.load_file file
-      item_hash = HashConverter.hash_to_bib hash
-      bib = ::RelatonIsoBib::IsoBibliographicItem.new(**item_hash)
-      if edition_greater?(doc, bib) || replace_substage98?(doc, bib)
-        write_file file, doc, docid
-      elsif @files.include?(file) && !edition_greater?(bib, doc)
-        Util.warn "Duplicate file `#{file}` for `#{docid.id}` from #{url(docpath)}"
-      end
-    end
-
-    def edition_greater?(doc, bib)
-      doc.edition && bib.edition && doc.edition.content.to_i > bib.edition.content.to_i
-    end
-
-    def replace_substage98?(doc, bib) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-      doc.edition&.content == bib.edition&.content &&
-        (doc.status&.substage&.value != "98" || bib.status&.substage&.value == "98")
-    end
-
-    def write_file(file, doc, docid)
-      @files << file
-      index.add_or_update docid.to_h, file
-      File.write file, serialize(doc), encoding: "UTF-8"
-    end
-
-    #
-    # Serialize document to string.
-    #
-    # @param [RelatonIsoBib::IsoBibliographicItem] doc document
-    #
-    # @return [String] serialized document
-    #
-    def serialize(doc)
-      case @format
-      when "yaml" then doc.to_hash.to_yaml
-      when "bibxml" then doc.to_bibxml
-      when "xml" then doc.to_xml bibdata: true
-      end
-    end
-
-    private
-
-    #
-    # Create thread worker
-    #
-    # @return [Thread] thread
-    #
-    def thread
-      Thread.new do
-        while (path = @queue.pop) != :END
-          yield path
-        end
-      end
-    end
-  end
-end

data/lib/relaton_iso/document_identifier.rb

@@ -1,46 +0,0 @@
-module RelatonIso
-  class DocumentIdentifier < RelatonBib::DocumentIdentifier
-    def id # rubocop:disable Metrics/MethodLength
-      id_str = @id.to_s.sub(/\sED\d+/, "").squeeze(" ").sub(/^ISO\/\s/, "ISO ") # workarounds for pubid gem bugs
-      if @all_parts
-        if type == "URN"
-          return "#{@id.urn}:ser"
-        else
-          return "#{id_str} (all parts)"
-        end
-      end
-      type == "URN" ? @id.urn.to_s : id_str
-    rescue Pubid::Iso::Errors::NoEditionError => e
-      Util.warn "#{type} identifier can't be generated for `#{@id}`: #{e.message}"
-    end
-
-    def to_h
-      stringify_values(@id.to_h) if @id.respond_to? :to_h
-    end
-
-    def remove_part
-      @id.part = nil
-    end
-
-    def remove_date
-      @id.year = nil
-    end
-
-    def all_parts
-      @all_parts = true
-    end
-
-    def stringify_values(hash)
-      hash.transform_values { |v| stringify(v) }.reject { |_k, v| v.empty? }
-    end
-
-    def stringify(val)
-      case val
-      when Array then val.map { |i| i.is_a?(Hash) ? stringify_values(i) : i.to_s }
-      when Hash then stringify_values(val)
-      when Symbol then val
-      else val.to_s
-      end
-    end
-  end
-end

data/lib/relaton_iso/hash_converter.rb

@@ -1,15 +0,0 @@
-module RelatonIso
-  module HashConverter
-    include RelatonIsoBib::HashConverter
-    extend self
-
-    def create_docid(**args)
-      begin
-        args[:id] = Pubid::Iso::Identifier.parse args[:id] if args[:id].is_a?(String) && args[:primary]
-      rescue StandardError
-        Util.warn "Unable to create a Pubid::Iso::Identifier from `#{args[:id]}`"
-      end
-      DocumentIdentifier.new(**args)
-    end
-  end
-end

data/lib/relaton_iso/hit.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-module RelatonIso
-  # Hit.
-  class Hit < RelatonBib::Hit
-    # @return [RelatonIsoBib::IsoBibliographicItem]
-    attr_writer :fetch
-
-    # @return [Pubid::Iso::Identifier] pubid
-    attr_writer :pubid
-
-    # Update edition for pubid when provided in Bibliographic Item
-    # def update_edition(bibliographic_item)
-    #   if bibliographic_item.edition
-    #     pubid.root.edition = bibliographic_item.edition.content
-    #   end
-    # end
-
-    # Parse page.
-    # @param lang [String, nil]
-    # @return [RelatonIso::IsoBibliographicItem]
-    def fetch(_lang = nil)
-      @fetch ||= begin
-        url = "#{HitCollection::ENDPOINT}#{hit[:file]}"
-        resp = Net::HTTP.get_response URI(url)
-        hash = YAML.safe_load resp.body
-        hash["fetched"] = Date.today.to_s
-        RelatonIsoBib::IsoBibliographicItem.from_hash hash
-      end
-    end
-
-    # @return [Integer]
-    def sort_weight
-      case hit[:status] # && hit["publicationStatus"]["key"]
-      when "Published" then 0
-      when "Under development" then 1
-      when "Withdrawn" then 2
-      when "Deleted" then 3
-      else 4
-      end
-    end
-
-    # @return [Pubid::Iso::Identifier]
-    def pubid
-      return @pubid if defined? @pubid
-
-      create_pubid hit[:id]
-    rescue StandardError
-      Util.warn "Unable to create an identifier from #{hit[:id]}"
-      @pubid = nil
-    end
-
-    private
-
-    def create_pubid(id)
-      @pubid = id.is_a?(Hash) ? Pubid::Iso::Identifier.create(**id) : id
-    end
-  end
-end

data/lib/relaton_iso/hit_collection.rb

@@ -1,100 +0,0 @@
-# frozen_string_literal: true
-
-require "relaton_iso/hit"
-
-module RelatonIso
-  # Page of hit collection.
-  class HitCollection < RelatonBib::HitCollection
-    INDEXFILE = "index-v1.yaml"
-    ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-iso/main/"
-
-    # @param text [Pubid::Iso::Identifier] reference to search
-    def initialize(pubid, opts = {})
-      super
-      @opts = opts
-    end
-
-    # @return [Pubid::Iso::Identifier]
-    alias ref_pubid text
-
-    def ref_pubid_no_year
-      @ref_pubid_no_year ||= ref_pubid.dup.tap { |r| r.base = r.base.exclude(:year) if r.base }
-    end
-
-    def ref_pubid_excluded
-      @ref_pubid_excluded ||= ref_pubid_no_year.exclude(*excludings)
-    end
-
-    def fetch # rubocop:disable Metrics/AbcSize
-      @array = index.search do |row|
-        row[:id].is_a?(Hash) ? pubid_match?(row[:id]) : ref_pubid.to_s == row[:id]
-      end.map { |row| Hit.new row, self }
-        .sort_by! { |h| h.pubid.to_s }
-        .reverse!
-      self
-    end
-
-    def pubid_match?(id)
-      pubid = create_pubid(id)
-      return false unless pubid
-
-      pubid.base = pubid.base.exclude(:year, :edition) if pubid.base
-      dir_excludings = excludings.dup
-      dir_excludings << :edition unless pubid.typed_stage_abbrev == "DIR"
-      pubid.exclude(*dir_excludings) == ref_pubid_excluded
-    end
-
-    def create_pubid(id)
-      Pubid::Iso::Identifier.create(**id)
-    rescue StandardError => e
-      Util.warn e.message, key: ref_pubid.to_s
-    end
-
-    def excludings
-      return @excludings if defined? @excludings
-
-      excl_parts = %i[year]
-      excl_parts << :part if ref_pubid.root.part.nil? || @opts[:all_parts]
-      if ref_pubid.stage.nil? || @opts[:all_parts]
-        excl_parts << :stage
-        excl_parts << :iteration
-      end
-      # excl_parts << :edition if ref_pubid.root.edition.nil? || all_parts
-      @escludings = excl_parts
-    end
-
-    def index
-      @index ||= Relaton::Index.find_or_create :iso, url: "#{ENDPOINT}index-v1.zip", file: INDEXFILE
-    end
-
-    def fetch_doc
-      if !@opts[:all_parts] || size == 1
-        any? && first.fetch(@opts[:lang])
-      else
-        to_all_parts(@opts[:lang])
-      end
-    end
-
-    # @param lang [String, nil]
-    # @return [RelatonIsoBib::IsoBibliographicItem, nil]
-    def to_all_parts(lang = nil) # rubocop:disable Metrics/AbcSize
-      hit = @array.min_by { |h| h.pubid.part.to_i }
-      return @array.first&.fetch lang unless hit
-
-      bibitem = hit.fetch(lang)
-      all_parts_item = bibitem.to_all_parts
-      @array.reject { |h| h.pubid.part == hit.pubid.part }.each do |hi|
-        all_parts_item.relation << create_relation(hi)
-      end
-      all_parts_item
-    end
-
-    def create_relation(hit)
-      docid = DocumentIdentifier.new(id: hit.pubid, type: "ISO", primary: true)
-      isobib = RelatonIsoBib::IsoBibliographicItem.new(
-        formattedref: RelatonBib::FormattedRef.new(content: hit.pubid.to_s), docid: [docid],
-      )
-      RelatonBib::DocumentRelation.new(type: "instanceOf", bibitem: isobib)
-    end
-  end
-end