relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/index/shard_source.rb

@@ -0,0 +1,201 @@
+require "json"
+require "net/http"
+require "openssl"
+require "zlib"
+
+module Relaton
+  module Index
+    #
+    # Reads a flavor's machine index from its GitHub Pages site (relaton#189).
+    #
+    # `index/manifest.json` gives the lookup rule, and a query reads only the
+    # one `index/shard-NNNNN.json` its document family lives in, in place of the
+    # whole `index-vN.zip`. The shard key is `crc32(id.root.number.to_s) %
+    # shards`, the expression the relaton-cli producer (`MachineIndex`) writes
+    # and `Type#candidates_by_number` narrows on. See
+    # `docs/data-repository-format.adoc`, "The machine index on the Pages site".
+    #
+    # Everything is held in memory for TTL seconds and never written to the
+    # storage. A shard 404 is a definitive "not found" (empty buckets are not
+    # published). A transport failure raises `Relaton::RequestError`, which
+    # `Relaton::Db#net_retry` retries; a machine index this client cannot read
+    # raises `Relaton::Index::Error`. Neither falls back to the monolith: the
+    # whole index is read only when the manifest publishes no shards, or when
+    # the query has no root number to key on.
+    #
+    class ShardSource
+      TTL = 24 * 60 * 60
+      VERSION = 2
+      KEY = "root-number".freeze
+      ALGORITHM = "crc32".freeze
+
+      NET_ERRORS = [
+        SocketError, Timeout::Error, IOError, SystemCallError, OpenSSL::SSL::SSLError,
+        Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
+        Zlib::Error # a truncated gzip body, which Net::HTTP inflates
+
+      ].freeze
+
+      attr_reader :pages_url
+
+      #
+      # @param [String] dir flavor directory name, used in log messages
+      # @param [String] pages_url base URL of the Pages site, with a trailing `/`
+      # @param [Class] pubid_class the flavor's pubid identifier class
+      #
+      def initialize(dir, pages_url, pubid_class)
+        @pages_url = pages_url.end_with?("/") ? pages_url : "#{pages_url}/"
+        # Only the deserialization helpers are used: this FileIO reads and
+        # writes no file.
+        @file_io = FileIO.new(dir, nil, nil, nil, pubid_class)
+        @mutex = Mutex.new
+        @state = nil
+      end
+
+      #
+      # Rows of the shard that holds the document family of `id`.
+      #
+      # @param [Pubid::Identifier] id parsed query
+      #
+      # @return [Array<Hash>] `{ id:, file: }` rows, empty when not found
+      #
+      def rows(id)
+        number = id.root.number.to_s
+        state = current_state
+        shards = manifest(state)["shards"]
+        return load_whole_index(state) if shards.zero? || number.empty?
+
+        n = Zlib.crc32(number) % shards
+        once(state, n) { fetch_shard(n) }
+      end
+
+      #
+      # Every row, from the monolith the manifest names (`<index>.zip` on the
+      # Pages site). Needed by a String query and a block-only search.
+      #
+      # @return [Array<Hash>] `{ id:, file: }` rows, sorted by root number
+      #
+      def whole_index
+        load_whole_index current_state
+      end
+
+      private
+
+      # One generation of cached data. It is replaced as a whole when it
+      # expires, so a fetch that finishes late stores into its own generation
+      # and never mixes an old shard with a new manifest.
+      def new_state
+        { created: Time.now, data: {}, locks: {} }
+      end
+
+      def current_state
+        @mutex.synchronize do
+          @state = new_state if @state.nil? || Time.now - @state[:created] > TTL
+          @state
+        end
+      end
+
+      # Runs the block once for each key in a generation. The network call is
+      # made under a lock for that key only, so lookups of other shards (from
+      # the `Relaton::Db` workers pool) do not wait for it. A raise stores
+      # nothing, so the next call tries again.
+      def once(state, key)
+        lock = @mutex.synchronize { state[:locks][key] ||= Mutex.new }
+        lock.synchronize do
+          data = state[:data]
+          return @mutex.synchronize { data[key] } if @mutex.synchronize { data.key?(key) }
+
+          value = yield
+          @mutex.synchronize { data[key] = value }
+        end
+      end
+
+      def manifest(state)
+        once(state, :manifest) do
+          body = get("index/manifest.json")
+          raise Error, "no machine index manifest at `#{url('index/manifest.json')}`" unless body
+
+          check_manifest parse_json(body, "index/manifest.json")
+        end
+      end
+
+      def check_manifest(data)
+        unless data.is_a?(Hash) && data["version"] == VERSION &&
+            data["key"] == KEY && data["algorithm"] == ALGORITHM &&
+            data["shards"].is_a?(Integer) && !data["shards"].negative? &&
+            data["index"].is_a?(String) && !data["index"].empty?
+          raise Error, "unsupported machine index manifest at " \
+                       "`#{url('index/manifest.json')}`: #{data.inspect}"
+        end
+
+        data
+      end
+
+      def fetch_shard(num)
+        path = format("index/shard-%05d.json", num)
+        body = get(path)
+        return [] unless body
+
+        data = parse_json(body, path)
+        raise Error, "shard `#{url(path)}` is not an array" unless data.is_a?(Array)
+
+        data.map { |r| { id: deserialize(r["id"], path), file: r["file"] } }
+      end
+
+      def load_whole_index(state)
+        path = "#{manifest(state)['index']}.zip"
+        once(state, :whole_index) do
+          body = get(path)
+          raise Error, "no whole index at `#{url(path)}`" unless body
+
+          yaml = unzip(body)
+          raise Error, "empty archive at `#{url(path)}`" unless yaml
+
+          index = YAML.safe_load(yaml, permitted_classes: [Symbol])
+          raise Error, "wrong structure of `#{url(path)}`" unless @file_io.check_basic_format(index)
+
+          @file_io.deserialize_pubid(index)
+        rescue FileIO::InvalidIndexError, Psych::SyntaxError, Zip::Error, Zlib::Error => e
+          raise Error, "cannot read `#{url(path)}`: #{e.message}"
+        end
+      end
+
+      def unzip(body)
+        yaml = nil
+        # `open_buffer` does not return the block's value.
+        Zip::File.open_buffer(body) { |zip| yaml = zip.entries.first&.get_input_stream&.read }
+        yaml
+      end
+
+      def deserialize(raw, path)
+        @file_io.deserialize_id(raw)
+      rescue FileIO::InvalidIndexError => e
+        raise Error, "cannot read shard `#{url(path)}`: #{e.message}"
+      end
+
+      def parse_json(body, path)
+        JSON.parse(body)
+      rescue JSON::ParserError => e
+        raise Error, "cannot parse `#{url(path)}`: #{e.message}"
+      end
+
+      def url(path)
+        "#{@pages_url}#{path}"
+      end
+
+      #
+      # @return [String, nil] the body, or nil for a 404
+      #
+      def get(path)
+        resp = Net::HTTP.get_response(URI.parse(url(path)))
+        case resp.code
+        when "200" then resp.body
+        when "404" then nil
+        else raise Relaton::RequestError, "Could not access #{url(path)}: HTTP #{resp.code}"
+        end
+      rescue *NET_ERRORS => e
+        raise Relaton::RequestError, "Could not access #{url(path)}: #{e.message}"
+      end
+    end
+  end
+end

data/lib/relaton/index/type.rb

@@ -0,0 +1,210 @@
+module Relaton
+  module Index
+    #
+    # Relaton::Index::Type is a class for indexing Relaton files.
+    #
+    class Type
+      #
+      # Initialize a new Relaton::Index::Type object
+      #
+      # @param [String, Symbol] type type of index (ISO, IEC, etc.)
+      # @param [String, nil] url external URL to index, used to fetch index for searching files
+      # @param [String, nil] file output file name
+      # @param [Array<Symbol>] id_keys keys of identifier to be used for sorting index
+      #   format of index file is checked if id_keys all is provided at least in one of the IDs
+      # @param [Pubid::Identifier, nil] pubid class for deserialization
+      # @param [String, nil] pages_url base URL of the Pages site that serves
+      #   the machine index. With it the type reads the index from there, in
+      #   memory (see ShardSource), and a parsed query fetches one shard only.
+      #
+      def initialize(type, url = nil, file = nil, id_keys = nil, pubid_class = nil, # rubocop:disable Metrics/ParameterLists
+                     pages_url: nil)
+        @file = file
+        @dir = type.to_s.downcase
+        @pubid_class = pubid_class
+        @pages_url = pages_url
+        filename = file || Index.config.filename
+        @file_io = FileIO.new @dir, url, filename, id_keys, pubid_class
+        @source = new_source
+      end
+
+      # With a Pages source the whole index is not kept here: the source holds
+      # it, and drops it when its 24 h expire.
+      def index
+        return @source.whole_index if @source
+
+        @index ||= @file_io.read
+      end
+
+      #
+      # Check if index is actual. If url or file is given, check if it is equal to
+      # index url or file.
+      #
+      # @param [Hash] **args arguments
+      # @option args [String, nil] :url external URL to index, used to fetch index for searching files
+      # @option args [String, nil] :file output file name
+      # @option args [String, nil] :pages_url base URL of the Pages site
+      #
+      # @return [Boolean] true if index is actual, false otherwise
+      #
+      def actual?(**args)
+        (!args.key?(:url) || args[:url] == @file_io.url) &&
+          (!args.key?(:file) || args[:file] == @file) &&
+          (!args.key?(:pages_url) || args[:pages_url] == @pages_url)
+      end
+
+      #
+      # Add or update index item
+      #
+      # @param [Pubid::Identifier] id document ID
+      # @param [String] file file name of the document
+      #
+      # @return [void]
+      #
+      def add_or_update(id, file)
+        key = id.to_s
+        item = id_lookup[key]
+        if item
+          item[:file] = file
+        else
+          new_item = { id: id, file: file }
+          index << new_item
+          id_lookup[key] = new_item
+          @file_io.sorted = false
+        end
+      end
+
+      #
+      # Search index for a given ID.
+      #
+      # Without a block, two identifiers are matched with pubid's asymmetric
+      # subset match `id === item[:id]` (`Pubid::SubsetMatch`): the query is the
+      # reference, and a component it leaves nil or empty matches any value, so
+      # `OGC 12-128` answers with `12-128r19`. That is what a search by
+      # reference means. A caller that needs **exact** equality passes
+      # `exact: true`, which selects the rows with `item[:id] == id`. A nil
+      # component is a wildcard for `===`, not "the document has none": over
+      # the fixture indexes the subset match accepts 260 CCSDS pairs (the `-S`
+      # suffix) and 20,513 IETF pairs (a draft slug matches each of its
+      # versions) that `==` rejects. `Relaton::Ccsds` and `Relaton::Ietf` are
+      # the two callers that pass it.
+      #
+      # A String query keeps the substring match it always had, unless it is
+      # `exact:`.
+      #
+      # @param [String, Pubid::Identifier] id ID to search for
+      # @param [Boolean] exact match with `==` instead of the subset match
+      #
+      # @return [Array<Hash>] search results
+      #
+      def search(id = nil, exact: false, &block)
+        raise ArgumentError, "search takes exact: or a block, not both" if exact && block
+
+        items = search_candidates(id)
+        return items.select(&block) if block
+        return items.select { |i| i[:id] == id } if exact
+
+        items.select { |i| match_item(i, id) }
+      end
+
+      #
+      # Save index to storage
+      #
+      # @return [void]
+      #
+      def save
+        @file_io.save(@index || [])
+      end
+
+      #
+      # Remove index file from storage and clear index
+      #
+      # @return [void]
+      #
+      def remove_file
+        # A Pages source has no file: its FileIO would resolve the default
+        # filename against the working directory.
+        @file_io.remove unless @source
+        @source = new_source
+        @index = nil
+        @id_lookup = nil
+      end
+
+      #
+      # Remove all index items
+      #
+      # @return [void]
+      #
+      def remove_all
+        @index = []
+        @id_lookup = nil
+        @file_io.sorted = true
+      end
+
+      private
+
+      def id_lookup
+        @id_lookup ||= index.each_with_object({}) do |item, h|
+          h[item[:id].to_s] = item
+        end
+      end
+
+      def new_source
+        ShardSource.new(@dir, @pages_url, @pubid_class) if @pages_url
+      end
+
+      def search_candidates(id)
+        # A parsed query reads only its own shard. `@index`, once loaded, is
+        # never refreshed, so the shards stay the fresher answer.
+        return @source.rows(id) if @source && id && !id.is_a?(String)
+
+        # index needs to be created to check if sorted
+        idx = index
+        if @file_io.sorted && id && !id.is_a?(String)
+          candidates_by_number(id)
+        else
+          idx
+        end
+      end
+
+      # Narrowing key: the base *document's* number as a string. `#root` walks a
+      # supplement/amendment/corrigendum's `.base` chain to the origin (and
+      # returns self for a base document), so a document and all its wrappers
+      # share one key and cluster together. `.to_s` because the key is compared
+      # as a string (a pubid number Component is not `<`/`>`-comparable), and
+      # FileIO sorts the index by this exact same key so bsearch stays valid.
+      def candidates_by_number(id)
+        target = id.root.number.to_s
+        left = bsearch_left(target)
+        return [] unless left
+
+        right = bsearch_right(target)
+        index[left...right]
+      end
+
+      def bsearch_left(target)
+        index.bsearch_index do |item|
+          item[:id].root.number.to_s >= target
+        end
+      end
+
+      def bsearch_right(target)
+        index.bsearch_index do |item|
+          item[:id].root.number.to_s > target
+        end || index.size
+      end
+
+      # The query is the reference, so two identifiers take the subset match.
+      # See #search for the exact-equality escape hatch.
+      def match_item(item, id)
+        if item[:id].is_a?(String)
+          item[:id].include?(id.is_a?(String) ? id : id.to_s)
+        elsif id.is_a?(String)
+          item[:id].to_s.include?(id)
+        else
+          id === item[:id]
+        end
+      end
+    end
+  end
+end

data/lib/relaton/index/util.rb

@@ -0,0 +1,18 @@
+module Relaton
+  module Index
+    module Util
+      extend self
+
+      PROGNAME = "relaton-index".freeze
+
+      def method_missing(method_name, msg = nil, prog = nil, **opts, &block)
+        prog ||= self::PROGNAME
+        Relaton.logger_pool.send method_name, msg, prog, **opts, &block
+      end
+
+      def respond_to_missing?(method_name, include_private = false)
+        Relaton.logger_pool.respond_to?(method_name) || super
+      end
+    end
+  end
+end

data/lib/relaton/index.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "zip"
+require "relaton/logger"
+
+require_relative "version"
+require_relative "core/request_error"
+require_relative "index/file_storage"
+require_relative "index/config"
+require_relative "index/util"
+require_relative "index/pool"
+require_relative "index/type"
+require_relative "index/file_io"
+require_relative "index/shard_source"
+
+module Relaton
+  module Index
+    class Error < StandardError; end
+
+    class << self
+      #
+      # Proxy for Pool#type
+      #
+      def find_or_create(type, **args)
+        pool.type(type, **args)
+      end
+
+      def close(type)
+        pool.remove type
+      end
+
+      #
+      # Create new index pool object or return existing
+      #
+      # @return [Relaton::Index::Pool] index pool
+      #
+      def pool
+        @pool ||= Pool.new
+      end
+
+      #
+      # Create new config object or return existing
+      #
+      # @return [Relaton::Index::Config] config object
+      #
+      def config
+        @config ||= Config.new
+      end
+
+      #
+      # Configure Relaton::Index
+      #
+      # @return [void]
+      #
+      def configure
+        yield config
+      end
+    end
+  end
+end

data/lib/relaton/isbn/isbn.rb

@@ -0,0 +1,61 @@
+module Relaton
+  module Isbn
+    class Isbn
+      #
+      # Create ISBN object.
+      #
+      # @param [String] isbn ISBN 13 number
+      #
+      def initialize(isbn)
+        @isbn = isbn&.delete("-")&.sub(/^ISBN[\s:]/i, "")
+      end
+
+      def parse
+        convert_to13
+      end
+
+      def check?
+        case @isbn
+        when /^\d{9}[\dX]$/i then check10?
+        when /^\d{13}$/ then check13?
+        else false
+        end
+      end
+
+      def check10?
+        @isbn[9] == calc_check_digit10
+      end
+
+      def check13?
+        @isbn[12] == calc_check_digit13
+      end
+
+      def calc_check_digit10
+        sum = 0
+        @isbn[..-2].chars.each_with_index do |c, i|
+          sum += c.to_i * (10 - i)
+        end
+        chk = (11 - sum % 11) % 11
+        chk == 10 ? "X" : chk.to_s
+      end
+
+      def calc_check_digit13
+        sum = 0
+        @isbn[..-2].chars.each_with_index do |c, i|
+          sum += c.to_i * (i.even? ? 1 : 3)
+        end
+        ((10 - sum % 10) % 10).to_s
+      end
+
+      def convert_to13
+        return unless check?
+
+        return @isbn if @isbn.size == 13
+
+        @isbn = "978#{@isbn}"
+        @isbn[12] = calc_check_digit13
+        @isbn
+      end
+    end
+  end
+end

data/lib/relaton/isbn/open_library.rb

@@ -0,0 +1,43 @@
+module Relaton
+  module Isbn
+    #
+    # Search ISBN in Openlibrary.
+    #
+    module OpenLibrary
+      extend self
+
+      ENDPOINT = "http://openlibrary.org/api/volumes/brief/isbn/".freeze
+
+      def get(ref, _date = nil, _opts = {}) # rubocop:disable Metrics/MethodLength
+        Util.info "Fetching from OpenLibrary ...", key: ref
+
+        isbn = Isbn.new(ref).parse
+        unless isbn
+          Util.info "Incorrect ISBN.", key: ref
+          return
+        end
+
+        resp = request_api isbn
+        unless resp
+          Util.info "Not found.", key: ref
+          return
+        end
+
+        bib = Parser.parse resp
+        Util.info "Found: `#{bib.docidentifier.first.content}`", key: ref
+        bib
+      end
+
+      def request_api(isbn)
+        uri = URI "#{ENDPOINT}#{isbn}.json"
+        response = Net::HTTP.get_response uri
+        return unless response.is_a? Net::HTTPSuccess
+
+        data = JSON.parse response.body
+        return unless data["records"]&.any?
+
+        data["records"].first.last
+      end
+    end
+  end
+end

data/lib/relaton/isbn/parser.rb

@@ -0,0 +1,88 @@
+module Relaton
+  module Isbn
+    #
+    # OpenLibrary document parser.
+    #
+    class Parser
+      ATTRS = %i[fetched title docidentifier source contributor date place ext].freeze
+
+      def initialize(doc)
+        @doc = doc
+      end
+
+      def self.parse(doc)
+        new(doc).parse
+      end
+
+      def parse
+        args = ATTRS.each_with_object({}) { |a, h| h[a] = send(a) }
+        Bib::ItemData.new(**args)
+      end
+
+      private
+
+      def fetched = Date.today.to_s
+
+      def title
+        t = [Bib::Title.new(content: @doc["data"]["title"], type: "main")]
+        if @doc["data"]["subtitle"]
+          t << Bib::Title.new(content: @doc["data"]["subtitle"], type: "subtitle")
+        end
+        t
+      end
+
+      def docidentifier
+        isbn = @doc["details"]["bib_key"].split(":").last
+        [Bib::Docidentifier.new(content: isbn, type: "ISBN", primary: true)]
+      end
+
+      def source
+        [Bib::Uri.new(content: @doc["recordURL"], type: "src")]
+      end
+
+      def contributor
+        create_authors + creaate_publishers
+      end
+
+      def create_authors
+        @doc["data"]["authors"].map do |a|
+          name = Bib::FullName.new(
+            completename: Bib::LocalizedString.new(content: a["name"]),
+          )
+          person = Bib::Person.new(
+            name: name,
+            uri: a["url"] ? [Bib::Uri.new(content: a["url"])] : [],
+          )
+          Bib::Contributor.new(
+            person: person,
+            role: [Bib::Contributor::Role.new(type: "author")],
+          )
+        end
+      end
+
+      def creaate_publishers
+        @doc["data"]["publishers"].map do |p|
+          org = Bib::Organization.new(
+            name: [Bib::TypedLocalizedString.new(content: p["name"])],
+          )
+          Bib::Contributor.new(
+            organization: org,
+            role: [Bib::Contributor::Role.new(type: "publisher")],
+          )
+        end
+      end
+
+      def date
+        @doc["publishDates"].map { Bib::Date.new type: "published", at: _1 }
+      end
+
+      def place
+        @doc["data"]["publish_places"]&.map { Bib::Place.new city: _1["name"] }
+      end
+
+      def ext
+        Bib::Ext.new(flavor: "isbn")
+      end
+    end
+  end
+end