relaton 3.0.0.pre.alpha.5 → 3.0.0.pre.alpha.7

data/lib/relaton/index/file_io.rb

@@ -11,6 +11,10 @@ module Relaton
       # wrong-structure handling (re-download, or stop and log).
       class InvalidIndexError < StandardError; end
 
+      # Bump when the sidecar payload shape changes, so an older sidecar is
+      # discarded and rebuilt instead of misread.
+      SIDECAR_VERSION = 2 # v2: the payload carries the yaml byte-size for freshness
+
       attr_reader :url, :pubid_class
       attr_accessor :sorted
 
@@ -288,11 +292,197 @@ module Relaton
       #
       def remove
         Index.config.storage.remove file
+        delete_sidecar
         []
       end
 
+      #
+      # Raw-row read for the lazy search path (relaton#242 stopgap): returns
+      # precomputed root-number sort keys and the rows as plain hashes — no
+      # pubid objects. A Marshal sidecar next to the yaml holds the keys+rows
+      # so repeat loads skip the YAML parse and the one-time full
+      # materialization; the sidecar is rebuilt whenever the yaml is newer.
+      #
+      # @return [Array<Array<String>, Array<Hash>] sort keys and raw rows
+      #
+      def read_raw
+        case url
+        when String
+          with_file_lock do
+            check_file ? read_raw_file : fetch_raw_and_save
+          end
+        else
+          read_raw_file || [[], []]
+        end
+      end
+
+      # Deserialize raw rows into pubid rows. Only the caller knows which
+      # slice it needs, so materialization happens here rather than at load.
+      #
+      # @param [Array<Hash>] rows raw rows
+      # @return [Array<Hash>] rows with deserialized ids
+      def materialize(rows)
+        return rows unless @pubid_class
+
+        rows.map { |r| { id: deserialize_id(r[:id]), file: r[:file] } }
+      end
+
+      # Save raw rows (possibly alongside the keys they were loaded with);
+      # sorts by the same root-number key the object path sorts by.
+      #
+      # @param [Array<String>] keys sort keys, parallel to rows
+      # @param [Array<Hash>] rows raw rows
+      # @return [void]
+      def save_raw(keys, rows)
+        ordered = @pubid_class ? keys.zip(rows).sort_by { |k, _| k.to_s }
+                                     .map { |_, r| r } : rows
+        yaml = ordered.map do |item|
+          { id: item[:id], file: item[:file] }
+        end.to_yaml
+        Index.config.storage.write file, yaml
+        delete_sidecar
+      end
+
       private
 
+      def sidecar_file
+        "#{file}.ms"
+      end
+
+      def delete_sidecar
+        File.delete(sidecar_file) if File.file?(sidecar_file)
+      rescue Errno::EACCES
+        nil
+      end
+
+      def read_raw_file
+        # The sidecar is authoritative while it describes the yaml byte-for-byte
+        # (same size) — the yaml is not even parsed until the sidecar is stale
+        # or unreadable.
+        if sidecar_fresh?
+          loaded = sidecar
+          return loaded if loaded
+        end
+
+        yaml = Index.config.storage.read(file)
+        return unless yaml
+
+        begin
+          raw = YAML.safe_load(yaml, permitted_classes: [Symbol])
+        rescue Psych::SyntaxError
+          warn_local_index_error("YAML parsing error when reading")
+          delete_sidecar
+          return [[], []]
+        end
+        build_raw(raw)
+      end
+
+      # Size, not mtime: NTFS timestamp coarseness makes a rewritten yaml
+      # carry the sidecar's own mtime, so "yaml is newer" misses the rebuild.
+      def sidecar_fresh?
+        File.file?(sidecar_file) && File.file?(file) &&
+          File.size(file) == sidecar_yaml_size
+      end
+
+      def sidecar_yaml_size
+        Marshal.load(File.binread(sidecar_file))[4]
+      rescue TypeError, ArgumentError, EOFError
+        nil
+      end
+
+      def sidecar
+        version, sorted, keys, rows = Marshal.load(File.binread(sidecar_file))
+        return unless version == SIDECAR_VERSION
+
+        @sorted = sorted
+        [keys, rows]
+      rescue TypeError, ArgumentError, EOFError
+        nil
+      end
+
+      def build_raw(raw)
+        unless check_format(raw)
+          warn_local_index_error("Wrong structure of")
+          return [[], []]
+        end
+
+        return build_raw_in_child(raw) if build_sidecar_in_child?
+
+        build_raw_in_process(raw)
+      rescue InvalidIndexError
+        warn_local_index_error("Wrong structure of")
+        [[], []]
+      end
+
+      # The one-time key build materializes every row (~675 MB on the
+      # 79,993-row ISO index). Freed pages are often not returned to the OS,
+      # so an in-process build leaves the parent's RSS high — and containers
+      # OOM on RSS (relaton#242). Where fork exists, build the sidecar in a
+      # child: the graph dies with it and the parent never spikes.
+      def build_sidecar_in_child?
+        Process.respond_to?(:fork) && Index.config.build_sidecar_in_child != false
+      end
+
+      def build_raw_in_child(raw)
+        pid = Process.fork do
+          build_raw_in_process(raw)
+          exit!(0)
+        end
+        Process.wait(pid)
+        loaded = sidecar
+        return build_raw_in_process(raw) unless loaded # child failed
+
+        loaded
+      rescue Errno::ENOMEM, SystemCallError
+        build_raw_in_process(raw)
+      end
+
+      def build_raw_in_process(raw)
+        objects = deserialize_pubid(raw)
+        keys = objects.map { |r| r[:id].root.number.to_s }
+        rows = objects.map { |r| { id: raw_id(r), file: r[:file] } }
+        write_sidecar(keys, rows)
+        [keys, rows]
+      rescue InvalidIndexError
+        warn_local_index_error("Wrong structure of")
+        [[], []]
+      end
+
+      def raw_id(row)
+        id = row[:id]
+        id.respond_to?(:to_hash) ? id.to_hash : id
+      end
+
+      def write_sidecar(keys, rows)
+        File.binwrite(sidecar_file,
+                      Marshal.dump([SIDECAR_VERSION, @sorted, keys, rows,
+                                    File.size(file)]))
+      rescue Errno::EACCES, Errno::ENOENT, Errno::EROFS
+        nil # the sidecar is an optimization; a read-only dir still works
+      end
+
+      def fetch_raw_and_save
+        uri = URI.parse(url)
+        body = Net::HTTP.get(uri)
+        yaml = nil
+        Zip::File.open_buffer(body) do |zip|
+          yaml = zip.entries.first.get_input_stream.read
+        end
+        Util.info "Downloaded index from `#{url}`", progname
+        raw = YAML.safe_load(yaml, permitted_classes: [Symbol])
+        if check_format(raw)
+          save raw
+          raw = nil # release the parsed copy; read_raw_file loads the sidecar's
+          read_raw_file
+        else
+          warn_remote_index_error "Wrong structure of"
+          [[], []]
+        end
+      rescue Psych::SyntaxError
+        warn_remote_index_error "YAML parsing error when reading"
+        [[], []]
+      end
+
       def with_file_lock(&)
         @@file_locks_mutex.synchronize do
           @@file_locks[file] ||= Mutex.new

data/lib/relaton/index/type.rb

@@ -30,7 +30,24 @@ module Relaton
       def index
         return @source.whole_index if @source
 
-        @index ||= @file_io.read
+        @index ||= @file_io.materialize(raw_index[1])
+      end
+
+      # Raw rows plus their precomputed root-number sort keys (relaton#242
+      # stopgap). The lazy search path materializes pubid objects only for
+      # the bucket it narrows to, so a process that answers one lookup holds
+      # raw hashes instead of the whole pubid graph. A caller-seeded `@index`
+      # (a spec fixture, a preloaded pool entry) is the source of truth: the
+      # raw side derives from it, in its order, so a seeder that sorts by the
+      # narrowing key keeps the bsearch valid.
+      def raw_index
+        @raw_index ||= if @index
+                          keys = @index.map { |r| narrowing_key(r[:id]) }
+                          rows = @index.map { |r| { id: raw_row_id(r[:id]), file: r[:file] } }
+                          [keys, rows]
+                        else
+                          @file_io.read_raw
+                        end
       end
 
       #
@@ -59,14 +76,23 @@ module Relaton
       # @return [void]
       #
       def add_or_update(id, file)
-        key = id.to_s
-        item = id_lookup[key]
-        if item
-          item[:file] = file
+        # A seeded or materialized @index (spec fixtures, preloaded pool
+        # entries) and pubid-less indexes run the legacy object path
+        # bit-for-bit: their consumers match rows by object identity
+        # semantics (`==`, `exclude`, to_s dedup) that raw rows cannot
+        # reproduce. Only a pubid-backed FileIO-loaded index — the
+        # relaton#242 memory case — adds through raw rows.
+        return legacy_add(id, file) if @index || !@file_io.pubid_class
+
+        raw_hash = raw_row_id(id)
+        keys, rows = raw_index
+        pos = raw_position(raw_hash)
+        if pos
+          rows[pos] = { id: raw_hash, file: file }
         else
-          new_item = { id: id, file: file }
-          index << new_item
-          id_lookup[key] = new_item
+          rows << { id: raw_hash, file: file }
+          keys << narrowing_key(id)
+          @raw_lookup[raw_hash] = rows.size - 1
           @file_io.sorted = false
         end
       end
@@ -110,7 +136,12 @@ module Relaton
       # @return [void]
       #
       def save
-        @file_io.save(@index || [])
+        if @index
+          @file_io.save(@index)
+        else
+          keys, rows = raw_index
+          @file_io.save_raw(keys, rows)
+        end
       end
 
       #
@@ -125,6 +156,8 @@ module Relaton
         @source = new_source
         @index = nil
         @id_lookup = nil
+        @raw_index = nil
+        @raw_lookup = nil
       end
 
       #
@@ -135,6 +168,8 @@ module Relaton
       def remove_all
         @index = []
         @id_lookup = nil
+        @raw_index = [[], []]
+        @raw_lookup = {}
         @file_io.sorted = true
       end
 
@@ -146,6 +181,44 @@ module Relaton
         end
       end
 
+      # Position of a raw row by its id hash — the raw-side equivalent of
+      # `id_lookup`, built once, so a crawl's add_or_update stays O(1) per row.
+      def raw_position(raw_hash)
+        @raw_lookup ||= raw_index[1].each_with_object({}) do |row, h|
+          h[row[:id]] = h.key?(row[:id]) ? h[row[:id]] : rows_index_of(h, row)
+        end
+        @raw_lookup[raw_hash]
+      end
+
+      def rows_index_of(lookup, row)
+        raw_index[1].index { |r| r[:id].equal?(row[:id]) || r[:id] == row[:id] }
+      end
+      # The pre-#242 add path, kept verbatim for seeded/materialized and
+      # pubid-less indexes.
+      def legacy_add(id, file)
+        key = id.to_s
+        item = id_lookup[key]
+        if item
+          item[:file] = file
+        else
+          new_item = { id: id, file: file }
+          index << new_item
+          id_lookup[key] = new_item
+          @file_io.sorted = false
+        end
+      end
+
+      # Same key expression the FileIO sidecar computes at build time; a
+      # String id (a no-pubid_class index) has no `.root`, and never narrows.
+      def narrowing_key(id)
+        id.respond_to?(:root) && id.root.respond_to?(:number) ? id.root.number.to_s : ""
+      end
+
+      def raw_row_id(id)
+        pc = @file_io.pubid_class
+        pc && id.is_a?(pc) ? id.to_hash : id
+      end
+
       def new_source
         ShardSource.new(@dir, @pages_url, @pubid_class) if @pages_url
       end
@@ -155,13 +228,15 @@ module Relaton
         # never refreshed, so the shards stay the fresher answer.
         return @source.rows(id) if @source && id && !id.is_a?(String)
 
-        # index needs to be created to check if sorted
-        idx = index
-        if @file_io.sorted && id && !id.is_a?(String)
-          candidates_by_number(id)
-        else
-          idx
+        # The lazy path narrows on the precomputed keys and materializes only
+        # the bucket; the materialized `index` stays available to callers
+        # that need the whole graph (`#index`, a String query, a block).
+        # Load first: a pubid-backed load is what settles @file_io.sorted.
+        if id && !id.is_a?(String) && @file_io.pubid_class
+          raw_index
+          return candidates_by_number(id) if @file_io.sorted
         end
+        index
       end
 
       # Narrowing key: the base *document's* number as a string. `#root` walks a
@@ -169,26 +244,24 @@ module Relaton
       # returns self for a base document), so a document and all its wrappers
       # share one key and cluster together. `.to_s` because the key is compared
       # as a string (a pubid number Component is not `<`/`>`-comparable), and
-      # FileIO sorts the index by this exact same key so bsearch stays valid.
+      # FileIO computes this exact same key once at sidecar-build time so the
+      # bsearch over the keys array stays valid.
       def candidates_by_number(id)
         target = id.root.number.to_s
+        keys, rows = raw_index
         left = bsearch_left(target)
         return [] unless left
 
         right = bsearch_right(target)
-        index[left...right]
+        @file_io.materialize(rows[left...right])
       end
 
       def bsearch_left(target)
-        index.bsearch_index do |item|
-          item[:id].root.number.to_s >= target
-        end
+        raw_index[0].bsearch_index { |k| k >= target }
       end
 
       def bsearch_right(target)
-        index.bsearch_index do |item|
-          item[:id].root.number.to_s > target
-        end || index.size
+        raw_index[0].bsearch_index { |k| k > target } || raw_index[0].size
       end
 
       # The query is the reference, so two identifiers take the subset match.

data/lib/relaton/iso/bibliography.rb

@@ -24,7 +24,9 @@ module Relaton
         raise Relaton::RequestError, e.message
       end
 
-      # @param ref [String] the ISO standard Code to look up (e..g "ISO 9000")
+      # @param ref [String, Pubid::Iso::Identifier] the ISO standard Code to
+      #   look up (e..g "ISO 9000"), or the parse that Relaton::Db routed with
+      #   (relaton#205); a pubid is never mutated
       # @param year [String, NilClass] the year the standard was published
       # @param opts [Hash] options; restricted to :all_parts if all-parts
       # @option opts [Boolean] :all_parts if all-parts reference is required
@@ -33,21 +35,21 @@ module Relaton
       #
       # @return [RelatonIsoBib::IsoBibliographicItem] Bibliographic item
       def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity,Metrics/AbcSize
-        code = ref.gsub("\u2013", "-")
-
-        # parse "all parts" request
-        # code.sub! " (all parts)", ""
-        # opts[:all_parts] ||= $~ && opts[:all_parts].nil?
-
-        query_pubid = ::Pubid::Iso::Identifier.parse(code)
+        query_pubid = if ref.is_a?(String)
+                        ::Pubid::Iso::Identifier.parse(ref.gsub("\u2013", "-"))
+                      else
+                        ref
+                      end
         if year&.respond_to?(:to_i)
+          # Copy first: a caller's pubid is also its Db cache key.
+          query_pubid = query_pubid.class.from_hash(query_pubid.to_hash)
           query_pubid.root.date = ::Pubid::Components::Date.new(year: year.to_s)
         end
         query_pubid = query_pubid.to_all_parts if opts[:all_parts]
         Util.info "Fetching from Relaton repository ...", key: query_pubid.to_s
 
         hits, missed_year_ids = isobib_search_filter(query_pubid, opts)
-        tip_ids = look_up_with_any_types_stages(hits, ref, opts)
+        tip_ids = look_up_with_any_types_stages(hits, ref.to_s, opts)
 
         date_filter = opts[:publication_date_before] || opts[:publication_date_after]
         if date_filter && !query_pubid.all_parts

data/lib/relaton/itu/bibliography.rb

@@ -31,13 +31,18 @@ module Relaton
         HitCollection.new(refid).tap(&:search)
       end
 
-      # @param code [String] the ITU standard Code to look up
+      # @param ref [String, Pubid::Itu::Identifier] the ITU standard Code to
+      #   look up, or its parse from Relaton::Db (relaton#205); #with_year
+      #   copies, so the pubid is not changed
       # @param year [String] the year the standard was published (optional)
       # @param opts [Hash] options
       # @return [Relaton::Bib::ItemData, nil]
-      def get(code, year = nil, opts = {})
-        warn_incorrect_ref(code)
-        refid = with_year ::Pubid::Itu.parse(code), year
+      def get(ref, year = nil, opts = {})
+        if ref.is_a? String
+          warn_incorrect_ref(ref)
+          ref = ::Pubid::Itu.parse(ref)
+        end
+        refid = with_year ref, year
 
         ret = itubib_get1(refid)
         return nil if ret.nil?

data/lib/relaton/jis/bibliography.rb

@@ -13,7 +13,8 @@ module Relaton
       # the reference's series and number (a supplement is filed under its base
       # number), so an edition and its amendments are returned together.
       #
-      # @param [String] code JIS document code
+      # @param [String, Pubid::Jis::Identifier] ref JIS document reference; a
+      #   pubid is never mutated
       # @param [String, nil] year JIS document year
       #
       # @return [Relaton::Jis::HitCollection] search result
@@ -21,16 +22,21 @@ module Relaton
       # @raise [Pubid::Errors::ParseError] when the reference is not a JIS
       #   identifier, so a malformed reference is not reported as "not found"
       #
-      def search(code, year = nil)
-        pubid = ::Pubid::Jis::Identifier.parse code
-        pubid.year ||= year.to_i if year
+      def search(ref, year = nil)
+        pubid = ref.is_a?(String) ? ::Pubid::Jis::Identifier.parse(ref) : ref
+        if year && pubid.year.nil?
+          # Copy first: a caller's pubid is also its Db cache key.
+          pubid = pubid.class.from_hash(pubid.to_hash)
+          pubid.year = year.to_i
+        end
         HitCollection.new pubid
       end
 
       #
       # Get JIS document by reference
       #
-      # @param [String] ref JIS document reference
+      # @param [String, Pubid::Jis::Identifier] ref JIS document reference, or
+      #   the parse that Relaton::Db routed with (relaton#205)
       # @param [String, nil] year JIS document year
       # @param [Hash] opts options
       # @option opts [Boolean] :all_parts return all parts of document
@@ -41,16 +47,21 @@ module Relaton
       #   identifier
       #
       def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize
-        code = ref.sub(/\s\((all parts|規格群)\)/, "")
-        opts[:all_parts] ||= !$1.nil?
-        Util.info "Fetching from webdesk.jsa.or.jp ...", key: ref
-        hits = search(code, year)
+        if ref.is_a?(String)
+          query = ref.sub(/\s\((all parts|規格群)\)/, "")
+          opts[:all_parts] ||= !$1.nil?
+        else
+          opts[:all_parts] ||= ref.all_parts?
+          query = ref.all_parts? ? ref.identifiers.first : ref
+        end
+        Util.info "Fetching from webdesk.jsa.or.jp ...", key: ref.to_s
+        hits = search(query, year)
         result = opts[:all_parts] ? hits.find_all_parts : hits.find
         if result.is_a? Bib::ItemData
-          Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
+          Util.info "Found: `#{result.docidentifier[0].content}`", key: ref.to_s
           return result
         end
-        hint result, ref, year
+        hint result, ref.to_s, year
       end
 
       #

data/lib/relaton/nist/bibliography.rb

@@ -7,18 +7,18 @@ module Relaton
         #
         # Search NIST documents by reference
         #
-        # @param text [String] reference
+        # @param ref [String] reference
         #
         # @return [Relaton::Nist::HitCollection] search result
         #
-        def search(text, year = nil, opts = {})
-          ref = text.sub(/^NISTIR/, "NIST IR").sub(/\/Add/, " Add")
+        def search(ref, year = nil, opts = {})
+          query = ref.sub(/^NISTIR/, "NIST IR").sub(/\/Add/, " Add")
           # pubid 2.x only recognizes the addendum marker with a trailing
           # period, and only splits an uppercase part letter — canonicalize
           # both ("800-38a Add" -> "800-38A Add.") so @reference parses to the
           # same pubid the index/CSRC carry.
-          ref = ref.sub(/\bAdd\b\.?/i, "Add.").sub(/([0-9])([a-z])(?=\s+Add\.)/) { "#{$1}#{$2.upcase}" }
-          HitCollection.search ref, year, opts
+          query = query.sub(/\bAdd\b\.?/i, "Add.").sub(/([0-9])([a-z])(?=\s+Add\.)/) { "#{$1}#{$2.upcase}" }
+          HitCollection.search query, year, opts
         rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError => e
           raise Relaton::RequestError, e.message
         end
@@ -26,34 +26,38 @@ module Relaton
         #
         # Get NIST document by reference
         #
-        # @param code [String] the NIST standard Code to look up
+        # @param ref [String, Pubid::Nist::Identifier] the NIST standard Code
+        #   to look up, or its parse from Relaton::Db (relaton#205)
         # @param year [String] the year the standard was published (optional)
         # @param opts [Hash] options
         # @option opts [Boolean] :all_parts restricted to all parts
         #
         # @return [Relaton::Nist::ItemData, nil] bibliographic item
         #
-        def get(code, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-          return fetch_ref_err(code, year, []) if code.match?(/\sEP$/)
+        def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          # The lookup below works on text, so a pubid from Relaton::Db is
+          # read back as its printed form; the pubid itself is not changed.
+          ref = ref.to_s unless ref.is_a?(String)
+          return fetch_ref_err(ref, year, []) if ref.match?(/\sEP$/)
 
-          /^(?<code2>[^(]+)(?:\((?<date2>\w+\s(?:\d{2},\s)?\d{4})\))?\s?\(?(?:(?<=\()(?<stage>(?:I|F|\d)PD))?/ =~ code
-          stage ||= /(?<=\.)PD-\w+(?=\.)/.match(code)&.to_s
+          /^(?<code2>[^(]+)(?:\((?<date2>\w+\s(?:\d{2},\s)?\d{4})\))?\s?\(?(?:(?<=\()(?<stage>(?:I|F|\d)PD))?/ =~ ref
+          stage ||= /(?<=\.)PD-\w+(?=\.)/.match(ref)&.to_s
           if code2
-            code = code2.strip
+            ref = code2.strip
             opts[:date] = parse_date(date2, str: false) if date2
             opts[:stage] = stage if stage
           end
 
           if year.nil?
-            /^(?<code1>[^:]+):(?<year1>[^:]+)$/ =~ code
+            /^(?<code1>[^:]+):(?<year1>[^:]+)$/ =~ ref
             unless code1.nil?
-              code = code1
+              ref = code1
               year = year1
             end
           end
 
-          code += "-1" if opts[:all_parts]
-          nistbib_get(code, year, opts)
+          ref += "-1" if opts[:all_parts]
+          nistbib_get(ref, year, opts)
         end
 
         private
@@ -61,14 +65,14 @@ module Relaton
         #
         # Get NIST document by reference
         #
-        # @param [String] code reference
+        # @param [String] ref reference
         # @param [String] year year
         # @param [Hash] opts options
         #
         # @return [Relaton::Nist::ItemData, nil] bibliographic item
         #
-        def nistbib_get(code, year, opts)
-          result = nistbib_search_filter(code, year, opts) || (return nil)
+        def nistbib_get(ref, year, opts)
+          result = nistbib_search_filter(ref, year, opts) || (return nil)
           ret = nistbib_results_filter(result, year, opts)
           if ret[:ret]
             Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: result.reference
@@ -153,14 +157,14 @@ module Relaton
         #
         # Get search results and filter them by code and year
         #
-        # @param code [String] reference
+        # @param ref [String] reference
         # @param year [String, nil] year
         # @param opts [Hash] options
         #
         # @return [Relaton::Nist::HitCollection] hits collection
         #
-        def nistbib_search_filter(code, year, opts)
-          result = search(code, year, opts)
+        def nistbib_search_filter(ref, year, opts)
+          result = search(ref, year, opts)
           result.search_filter
         end