relaton 3.0.0.pre.alpha.6 → 3.0.0.pre.alpha.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5029468b22fa9cc5e65bf7d76f8bb6db597fb331bc1f0a89a21186063ea49963
-  data.tar.gz: 6e572768f9ae838d04e682eb6e5cbc2a48dcb9e999fd2f4dce588952072475a6
+  metadata.gz: 21f068205ef6a80e718953c7989c7a51219a9f5f6e25a4fecc67ac93d9fa4da5
+  data.tar.gz: 7c438465496dc3b95370bc6990420073c63f41d5edba70022a71b39bd45f343f
 SHA512:
-  metadata.gz: 9ef70c48d70df0a8d7105b7195e0124f51a501eab815bca1ce6825a9113d338f3a1203c237d896443ae0f5ed1893efef97719bfd9a2938c53b58155171bf6c46
-  data.tar.gz: bb362dea5df12c9021b0725132e4f365aa8f97d6a81e9b41c2190c66121b6948aa4f6aee9a52f0c51e218ea38b5655b6ad014ea97d8100f00a83d07098ebfa0c
+  metadata.gz: af94d0ffa5ae9cbeefecf98d9aad8a04a7ea74814b92e470b6235b2bb1c2f19593db50e0f919b2ce3bbf1bdf44b8e637fa20000e6d59bc400736159fe0f9a2d1
+  data.tar.gz: 833bbe454eaa888a145fb9dcd5daff7d8bc8cf2e03f603c2100b0d19d981531a5b4bfce87b77bc00f3b5a5684904473f58656ebf820b2c7272b606f5c26afddc

data/lib/relaton/etsi/data_fetcher.rb

@@ -45,22 +45,117 @@ module Relaton
         report_errors
       end
 
-      def fetch_remaining_pages(first_page)
+      #
+      # Fetch pages 2..N. A page that stays bad after its retries is fetched
+      # again after the last page; if it is still bad, the crawl fails. A page
+      # is never skipped: relaton-data-etsi deletes data/ before the crawl and
+      # commits what it writes, so a skipped page would unpublish its documents.
+      #
+      # N comes from page 1's total_count, but the result set is sorted by
+      # deliverable number, so a document published during the crawl shifts the
+      # later pages by one. The crawl therefore reads on past N while the pages
+      # stay full (at most MAX_EXTRA_PAGES), and one page past a deferred last
+      # page; a record read twice overwrites its own file.
+      #
+      def fetch_remaining_pages(first_page) # rubocop:disable Metrics/MethodLength
+        total_pages = last = last_page(first_page)
+        deferred = []
+        page = 2
+        while page <= last
+          check_extra_pages page, total_pages
+          size = fetch_next_page(page, deferred)
+          break if size&.zero?
+
+          last = page + 1 if page == last && read_on?(size, page, total_pages)
+          page += 1
+        end
+        fetch_deferred_pages(deferred, total_pages)
+      end
+
+      def last_page(first_page)
         total = first_page.first ? first_page.first["total_count"].to_i : 0
-        total_pages = (total / PAGE_SIZE.to_f).ceil
-        (2..total_pages).each do |page|
-          records = fetch_page(page)
-          break if records.empty?
+        last = (total / PAGE_SIZE.to_f).ceil
+        first_page.size >= PAGE_SIZE ? [last, 2].max : last
+      end
+
+      #
+      # Whether to read the page after the last one: after a full page, or
+      # after a deferred page inside the total_count range. A deferred page
+      # past that range does not extend it, so a server that answers bad
+      # bodies past the end cannot keep the crawl going.
+      #
+      def read_on?(size, page, total_pages)
+        size ? size >= PAGE_SIZE : page <= total_pages
+      end
+
+      def check_extra_pages(page, total_pages)
+        return if page <= total_pages + MAX_EXTRA_PAGES
+
+        raise BadPage, "ETSI page #{page} is still full #{MAX_EXTRA_PAGES} " \
+                       "pages past total_count (#{total_pages} pages)."
+      end
+
+      #
+      # @return [Integer, nil] the number of records on the page; nil for a
+      #   deferred page
+      #
+      def fetch_next_page(page, deferred)
+        records = fetch_page(page)
+        process_records(records)
+        records.size
+      rescue BadPage => e
+        Util.warn "#{e.message} Fetching it again after the last page."
+        deferred << page
+        nil
+      end
 
-          process_records(records)
+      def fetch_deferred_pages(pages, total_pages)
+        return if pages.empty?
+
+        sleep DEFERRED_DELAY
+        failed = pages.filter_map do |page|
+          refetch_page page, total_pages
+        rescue BadPage => e
+          e.message
         end
+        raise BadPage, failed.join(" ") if failed.any?
+      end
+
+      # @return [String, nil] the failure message, nil on success
+      def refetch_page(page, total_pages)
+        records = fetch_page(page)
+        if records.empty? && page <= total_pages
+          return "ETSI page #{page} is empty on the re-fetch."
+        end
+
+        process_records records
+        nil
       end
 
       def fetch_page(page)
         date = Time.now.to_date + 1
         timestamp = (Time.now.to_f * 1000).to_i
         url = format(SOURCEURL, page: page, date: date, timestamp: timestamp)
-        JSON.parse(fetch_with_retry(url))
+        fetch_with_retry(url) { |body| parse_page(page, body) }
+      end
+
+      #
+      # ETSI's data.php sometimes answers with PHP's `Array` text in place of
+      # JSON (relaton-data-etsi crawl 36761804954); a re-fetch gets JSON.
+      #
+      # @raise [BadPage] when the body is not a JSON array
+      #
+      def parse_page(page, body)
+        records = JSON.parse(body)
+        return records if records.is_a?(Array)
+
+        raise BadPage, bad_page_message(page, body)
+      rescue JSON::ParserError
+        raise BadPage, bad_page_message(page, body)
+      end
+
+      def bad_page_message(page, body)
+        "ETSI page #{page} is not a JSON array: #{body.to_s[0, 200].inspect}."
       end
 
       def process_records(records)
@@ -92,16 +187,30 @@ module Relaton
         "Published"
       end
 
+      # A page body that is not a JSON array. See #parse_page.
+      class BadPage < StandardError; end
+
+      # Seconds to wait before a bad page is fetched again after the last page.
+      DEFERRED_DELAY = 60
+
+      # Full pages read past page 1's total_count before the crawl fails.
+      MAX_EXTRA_PAGES = 10
+
       NETWORK_ERRORS = [
         Mechanize::Error, Net::OpenTimeout, Net::ReadTimeout,
         SocketError, Errno::ECONNRESET
       ].freeze
 
+      #
+      # @yield [String] the body; the block's result is returned, and a BadPage
+      #   it raises is retried like a network error
+      #
       def fetch_with_retry(url, retries: 3, delay: 2) # rubocop:disable Metrics/MethodLength
         attempt = 0
         begin
-          Mechanize.new.get(url).body
-        rescue *NETWORK_ERRORS => e
+          body = Mechanize.new.get(url).body
+          block_given? ? yield(body) : body
+        rescue *NETWORK_ERRORS, BadPage => e
           attempt += 1
           if attempt <= retries
             Util.info "Fetch failed (#{e.message}), " \

data/lib/relaton/iec/processor.rb

@@ -7,7 +7,7 @@ module Relaton
         @short = :relaton_iec
         @prefix = "IEC"
         @pubid_flavor = :Iec # global prefixes sourced from Pubid::Iec.prefixes
-        @defaultprefix = %r{^(IEC\s|CISPR\s|IEV($|\s))}
+        @defaultprefix = %r{^(IEC\s|CISPR\s|IEV($|\s)|CEI\s)} # CEI: IEC's French spelling (relaton#243)
         @idtype = "IEC"
         @datasets = %w[iec-harmonized-all iec-harmonized-latest]
       end

data/lib/relaton/index/config.rb

@@ -4,7 +4,7 @@ module Relaton
     # Configuration class for Relaton::Index
     #
     class Config
-      attr_reader :storage, :storage_dir, :filename
+      attr_reader :storage, :storage_dir, :filename, :build_sidecar_in_child
 
       #
       # Set default values
@@ -13,6 +13,14 @@ module Relaton
         @storage = FileStorage
         @storage_dir = Dir.home
         @filename = "index.yaml"
+        @build_sidecar_in_child = true
+      end
+
+      # Build the sidecar in a forked child when available, so the one-time
+      # full materialization's memory dies with the child (relaton#242).
+      # Set to false to force an in-process build.
+      def build_sidecar_in_child=(flag)
+        @build_sidecar_in_child = flag
       end
 
       #

data/lib/relaton/index/file_io.rb

@@ -11,6 +11,10 @@ module Relaton
       # wrong-structure handling (re-download, or stop and log).
       class InvalidIndexError < StandardError; end
 
+      # Bump when the sidecar payload shape changes, so an older sidecar is
+      # discarded and rebuilt instead of misread.
+      SIDECAR_VERSION = 2 # v2: the payload carries the yaml byte-size for freshness
+
       attr_reader :url, :pubid_class
       attr_accessor :sorted
 
@@ -288,11 +292,197 @@ module Relaton
       #
       def remove
         Index.config.storage.remove file
+        delete_sidecar
         []
       end
 
+      #
+      # Raw-row read for the lazy search path (relaton#242 stopgap): returns
+      # precomputed root-number sort keys and the rows as plain hashes — no
+      # pubid objects. A Marshal sidecar next to the yaml holds the keys+rows
+      # so repeat loads skip the YAML parse and the one-time full
+      # materialization; the sidecar is rebuilt whenever the yaml is newer.
+      #
+      # @return [Array<Array<String>, Array<Hash>] sort keys and raw rows
+      #
+      def read_raw
+        case url
+        when String
+          with_file_lock do
+            check_file ? read_raw_file : fetch_raw_and_save
+          end
+        else
+          read_raw_file || [[], []]
+        end
+      end
+
+      # Deserialize raw rows into pubid rows. Only the caller knows which
+      # slice it needs, so materialization happens here rather than at load.
+      #
+      # @param [Array<Hash>] rows raw rows
+      # @return [Array<Hash>] rows with deserialized ids
+      def materialize(rows)
+        return rows unless @pubid_class
+
+        rows.map { |r| { id: deserialize_id(r[:id]), file: r[:file] } }
+      end
+
+      # Save raw rows (possibly alongside the keys they were loaded with);
+      # sorts by the same root-number key the object path sorts by.
+      #
+      # @param [Array<String>] keys sort keys, parallel to rows
+      # @param [Array<Hash>] rows raw rows
+      # @return [void]
+      def save_raw(keys, rows)
+        ordered = @pubid_class ? keys.zip(rows).sort_by { |k, _| k.to_s }
+                                     .map { |_, r| r } : rows
+        yaml = ordered.map do |item|
+          { id: item[:id], file: item[:file] }
+        end.to_yaml
+        Index.config.storage.write file, yaml
+        delete_sidecar
+      end
+
       private
 
+      def sidecar_file
+        "#{file}.ms"
+      end
+
+      def delete_sidecar
+        File.delete(sidecar_file) if File.file?(sidecar_file)
+      rescue Errno::EACCES
+        nil
+      end
+
+      def read_raw_file
+        # The sidecar is authoritative while it describes the yaml byte-for-byte
+        # (same size) — the yaml is not even parsed until the sidecar is stale
+        # or unreadable.
+        if sidecar_fresh?
+          loaded = sidecar
+          return loaded if loaded
+        end
+
+        yaml = Index.config.storage.read(file)
+        return unless yaml
+
+        begin
+          raw = YAML.safe_load(yaml, permitted_classes: [Symbol])
+        rescue Psych::SyntaxError
+          warn_local_index_error("YAML parsing error when reading")
+          delete_sidecar
+          return [[], []]
+        end
+        build_raw(raw)
+      end
+
+      # Size, not mtime: NTFS timestamp coarseness makes a rewritten yaml
+      # carry the sidecar's own mtime, so "yaml is newer" misses the rebuild.
+      def sidecar_fresh?
+        File.file?(sidecar_file) && File.file?(file) &&
+          File.size(file) == sidecar_yaml_size
+      end
+
+      def sidecar_yaml_size
+        Marshal.load(File.binread(sidecar_file))[4]
+      rescue TypeError, ArgumentError, EOFError
+        nil
+      end
+
+      def sidecar
+        version, sorted, keys, rows = Marshal.load(File.binread(sidecar_file))
+        return unless version == SIDECAR_VERSION
+
+        @sorted = sorted
+        [keys, rows]
+      rescue TypeError, ArgumentError, EOFError
+        nil
+      end
+
+      def build_raw(raw)
+        unless check_format(raw)
+          warn_local_index_error("Wrong structure of")
+          return [[], []]
+        end
+
+        return build_raw_in_child(raw) if build_sidecar_in_child?
+
+        build_raw_in_process(raw)
+      rescue InvalidIndexError
+        warn_local_index_error("Wrong structure of")
+        [[], []]
+      end
+
+      # The one-time key build materializes every row (~675 MB on the
+      # 79,993-row ISO index). Freed pages are often not returned to the OS,
+      # so an in-process build leaves the parent's RSS high — and containers
+      # OOM on RSS (relaton#242). Where fork exists, build the sidecar in a
+      # child: the graph dies with it and the parent never spikes.
+      def build_sidecar_in_child?
+        Process.respond_to?(:fork) && Index.config.build_sidecar_in_child != false
+      end
+
+      def build_raw_in_child(raw)
+        pid = Process.fork do
+          build_raw_in_process(raw)
+          exit!(0)
+        end
+        Process.wait(pid)
+        loaded = sidecar
+        return build_raw_in_process(raw) unless loaded # child failed
+
+        loaded
+      rescue Errno::ENOMEM, SystemCallError
+        build_raw_in_process(raw)
+      end
+
+      def build_raw_in_process(raw)
+        objects = deserialize_pubid(raw)
+        keys = objects.map { |r| r[:id].root.number.to_s }
+        rows = objects.map { |r| { id: raw_id(r), file: r[:file] } }
+        write_sidecar(keys, rows)
+        [keys, rows]
+      rescue InvalidIndexError
+        warn_local_index_error("Wrong structure of")
+        [[], []]
+      end
+
+      def raw_id(row)
+        id = row[:id]
+        id.respond_to?(:to_hash) ? id.to_hash : id
+      end
+
+      def write_sidecar(keys, rows)
+        File.binwrite(sidecar_file,
+                      Marshal.dump([SIDECAR_VERSION, @sorted, keys, rows,
+                                    File.size(file)]))
+      rescue Errno::EACCES, Errno::ENOENT, Errno::EROFS
+        nil # the sidecar is an optimization; a read-only dir still works
+      end
+
+      def fetch_raw_and_save
+        uri = URI.parse(url)
+        body = Net::HTTP.get(uri)
+        yaml = nil
+        Zip::File.open_buffer(body) do |zip|
+          yaml = zip.entries.first.get_input_stream.read
+        end
+        Util.info "Downloaded index from `#{url}`", progname
+        raw = YAML.safe_load(yaml, permitted_classes: [Symbol])
+        if check_format(raw)
+          save raw
+          raw = nil # release the parsed copy; read_raw_file loads the sidecar's
+          read_raw_file
+        else
+          warn_remote_index_error "Wrong structure of"
+          [[], []]
+        end
+      rescue Psych::SyntaxError
+        warn_remote_index_error "YAML parsing error when reading"
+        [[], []]
+      end
+
       def with_file_lock(&)
         @@file_locks_mutex.synchronize do
           @@file_locks[file] ||= Mutex.new

data/lib/relaton/index/type.rb

@@ -30,7 +30,24 @@ module Relaton
       def index
         return @source.whole_index if @source
 
-        @index ||= @file_io.read
+        @index ||= @file_io.materialize(raw_index[1])
+      end
+
+      # Raw rows plus their precomputed root-number sort keys (relaton#242
+      # stopgap). The lazy search path materializes pubid objects only for
+      # the bucket it narrows to, so a process that answers one lookup holds
+      # raw hashes instead of the whole pubid graph. A caller-seeded `@index`
+      # (a spec fixture, a preloaded pool entry) is the source of truth: the
+      # raw side derives from it, in its order, so a seeder that sorts by the
+      # narrowing key keeps the bsearch valid.
+      def raw_index
+        @raw_index ||= if @index
+                          keys = @index.map { |r| narrowing_key(r[:id]) }
+                          rows = @index.map { |r| { id: raw_row_id(r[:id]), file: r[:file] } }
+                          [keys, rows]
+                        else
+                          @file_io.read_raw
+                        end
       end
 
       #
@@ -59,14 +76,23 @@ module Relaton
       # @return [void]
       #
       def add_or_update(id, file)
-        key = id.to_s
-        item = id_lookup[key]
-        if item
-          item[:file] = file
+        # A seeded or materialized @index (spec fixtures, preloaded pool
+        # entries) and pubid-less indexes run the legacy object path
+        # bit-for-bit: their consumers match rows by object identity
+        # semantics (`==`, `exclude`, to_s dedup) that raw rows cannot
+        # reproduce. Only a pubid-backed FileIO-loaded index — the
+        # relaton#242 memory case — adds through raw rows.
+        return legacy_add(id, file) if @index || !@file_io.pubid_class
+
+        raw_hash = raw_row_id(id)
+        keys, rows = raw_index
+        pos = raw_position(raw_hash)
+        if pos
+          rows[pos] = { id: raw_hash, file: file }
         else
-          new_item = { id: id, file: file }
-          index << new_item
-          id_lookup[key] = new_item
+          rows << { id: raw_hash, file: file }
+          keys << narrowing_key(id)
+          @raw_lookup[raw_hash] = rows.size - 1
           @file_io.sorted = false
         end
       end
@@ -110,7 +136,12 @@ module Relaton
       # @return [void]
       #
       def save
-        @file_io.save(@index || [])
+        if @index
+          @file_io.save(@index)
+        else
+          keys, rows = raw_index
+          @file_io.save_raw(keys, rows)
+        end
       end
 
       #
@@ -125,6 +156,8 @@ module Relaton
         @source = new_source
         @index = nil
         @id_lookup = nil
+        @raw_index = nil
+        @raw_lookup = nil
       end
 
       #
@@ -135,6 +168,8 @@ module Relaton
       def remove_all
         @index = []
         @id_lookup = nil
+        @raw_index = [[], []]
+        @raw_lookup = {}
         @file_io.sorted = true
       end
 
@@ -146,6 +181,44 @@ module Relaton
         end
       end
 
+      # Position of a raw row by its id hash — the raw-side equivalent of
+      # `id_lookup`, built once, so a crawl's add_or_update stays O(1) per row.
+      def raw_position(raw_hash)
+        @raw_lookup ||= raw_index[1].each_with_object({}) do |row, h|
+          h[row[:id]] = h.key?(row[:id]) ? h[row[:id]] : rows_index_of(h, row)
+        end
+        @raw_lookup[raw_hash]
+      end
+
+      def rows_index_of(lookup, row)
+        raw_index[1].index { |r| r[:id].equal?(row[:id]) || r[:id] == row[:id] }
+      end
+      # The pre-#242 add path, kept verbatim for seeded/materialized and
+      # pubid-less indexes.
+      def legacy_add(id, file)
+        key = id.to_s
+        item = id_lookup[key]
+        if item
+          item[:file] = file
+        else
+          new_item = { id: id, file: file }
+          index << new_item
+          id_lookup[key] = new_item
+          @file_io.sorted = false
+        end
+      end
+
+      # Same key expression the FileIO sidecar computes at build time; a
+      # String id (a no-pubid_class index) has no `.root`, and never narrows.
+      def narrowing_key(id)
+        id.respond_to?(:root) && id.root.respond_to?(:number) ? id.root.number.to_s : ""
+      end
+
+      def raw_row_id(id)
+        pc = @file_io.pubid_class
+        pc && id.is_a?(pc) ? id.to_hash : id
+      end
+
       def new_source
         ShardSource.new(@dir, @pages_url, @pubid_class) if @pages_url
       end
@@ -155,13 +228,15 @@ module Relaton
         # never refreshed, so the shards stay the fresher answer.
         return @source.rows(id) if @source && id && !id.is_a?(String)
 
-        # index needs to be created to check if sorted
-        idx = index
-        if @file_io.sorted && id && !id.is_a?(String)
-          candidates_by_number(id)
-        else
-          idx
+        # The lazy path narrows on the precomputed keys and materializes only
+        # the bucket; the materialized `index` stays available to callers
+        # that need the whole graph (`#index`, a String query, a block).
+        # Load first: a pubid-backed load is what settles @file_io.sorted.
+        if id && !id.is_a?(String) && @file_io.pubid_class
+          raw_index
+          return candidates_by_number(id) if @file_io.sorted
         end
+        index
       end
 
       # Narrowing key: the base *document's* number as a string. `#root` walks a
@@ -169,26 +244,24 @@ module Relaton
       # returns self for a base document), so a document and all its wrappers
       # share one key and cluster together. `.to_s` because the key is compared
       # as a string (a pubid number Component is not `<`/`>`-comparable), and
-      # FileIO sorts the index by this exact same key so bsearch stays valid.
+      # FileIO computes this exact same key once at sidecar-build time so the
+      # bsearch over the keys array stays valid.
       def candidates_by_number(id)
         target = id.root.number.to_s
+        keys, rows = raw_index
         left = bsearch_left(target)
         return [] unless left
 
         right = bsearch_right(target)
-        index[left...right]
+        @file_io.materialize(rows[left...right])
       end
 
       def bsearch_left(target)
-        index.bsearch_index do |item|
-          item[:id].root.number.to_s >= target
-        end
+        raw_index[0].bsearch_index { |k| k >= target }
       end
 
       def bsearch_right(target)
-        index.bsearch_index do |item|
-          item[:id].root.number.to_s > target
-        end || index.size
+        raw_index[0].bsearch_index { |k| k > target } || raw_index[0].size
       end
 
       # The query is the reference, so two identifiers take the subset match.

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "3.0.0.pre.alpha.6".freeze
+  VERSION = "3.0.0.pre.alpha.7".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre.alpha.6
+  version: 3.0.0.pre.alpha.7
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-02 00:00:00.000000000 Z
+date: 2026-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable