relaton 3.0.0.pre.alpha.6 → 3.0.0.pre.alpha.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5029468b22fa9cc5e65bf7d76f8bb6db597fb331bc1f0a89a21186063ea49963
-  data.tar.gz: 6e572768f9ae838d04e682eb6e5cbc2a48dcb9e999fd2f4dce588952072475a6
+  metadata.gz: 3fafde9870593aca737727dcb4cf8cb142da74d255be4e855eed22bda4bd0f1b
+  data.tar.gz: 60986309859651e6e1baa05635a1ebfb2fec313f908a5508c14a59ded9ec3f8f
 SHA512:
-  metadata.gz: 9ef70c48d70df0a8d7105b7195e0124f51a501eab815bca1ce6825a9113d338f3a1203c237d896443ae0f5ed1893efef97719bfd9a2938c53b58155171bf6c46
-  data.tar.gz: bb362dea5df12c9021b0725132e4f365aa8f97d6a81e9b41c2190c66121b6948aa4f6aee9a52f0c51e218ea38b5655b6ad014ea97d8100f00a83d07098ebfa0c
+  metadata.gz: 2b075475ad0321b604674b8459c60ddc1e7ef10a1a0186e70042348c0d9d5bbe58166cb419548f8d98438040702ffd532204716a261bedaedf20888a5d6c08cf
+  data.tar.gz: 8b2b003b3b99918b96a42c622607feae3087e2778730a2d31dc94ebe9b51abea23a3f6fbdb254fc72a2daeeae47877feb4edb998d70aa6d0d46c86881db2812c

data/lib/relaton/bib/model/ics.rb

@@ -29,8 +29,7 @@ module Relaton
         return unless val.is_a?(String) && !val.empty?
         return if @text.is_a?(String) && !@text.empty?
 
-        description = Isoics.fetch(val)&.description
-        self.text = description if description
+        populate_text_from_isoics
       end
 
       # When the deserializer reaches the end of the XML element and
@@ -38,9 +37,44 @@ module Relaton
       # to mark the attribute as default-valued (suppressing serialization).
       # Refuse that mark if we've already populated text from Isoics so the
       # value survives round-trip. See #112.
+      # lutaml-model 0.8.92 (lutaml/lutaml-model#922): assignments made
+      # inside `code=` while from_xml is still mid-element do not register
+      # as explicit values. Populate once deserialization has finished.
+      def self.from_xml(node)
+        ics = super
+        # lutaml-model 0.8.92 (#922): a value assigned through a custom
+        # writer's `super` mid-deserialization is left default-suppressed.
+        # Register code as explicitly set, then populate text.
+        ics.value_set_for(:code) if ics.code.is_a?(String) && !ics.code.empty?
+        ics.populate_text_from_isoics
+        ics
+      end
+
+      def populate_text_from_isoics
+        code.is_a?(String) && !code.empty? or return
+        @text.is_a?(String) && !@text.empty? and return
+
+        description = Isoics.fetch(code)&.description
+        self.text = description if description
+      end
+
       def using_default_for(attribute_name)
-        return if attribute_name == :text &&
-          @text.is_a?(String) && !@text.empty?
+        # lutaml-model 0.8.92 (#922) leaves values assigned through a
+        # custom writer's `super` default-suppressed. code, when present,
+        # is always explicit; text absent from the XML is filled from
+        # Isoics here — the element end — and emitted.
+        if attribute_name == :code
+          return value_set_for(:code) if @code.is_a?(String) && !@code.empty?
+
+          return super
+        end
+
+        return if attribute_name == :text && @text.is_a?(String) && !@text.empty?
+
+        if attribute_name == :text
+          populate_text_from_isoics
+          return value_set_for(:text) unless @text.nil? || @text.empty?
+        end
 
         super
       end

data/lib/relaton/etsi/data_fetcher.rb

@@ -45,22 +45,117 @@ module Relaton
         report_errors
       end
 
-      def fetch_remaining_pages(first_page)
+      #
+      # Fetch pages 2..N. A page that stays bad after its retries is fetched
+      # again after the last page; if it is still bad, the crawl fails. A page
+      # is never skipped: relaton-data-etsi deletes data/ before the crawl and
+      # commits what it writes, so a skipped page would unpublish its documents.
+      #
+      # N comes from page 1's total_count, but the result set is sorted by
+      # deliverable number, so a document published during the crawl shifts the
+      # later pages by one. The crawl therefore reads on past N while the pages
+      # stay full (at most MAX_EXTRA_PAGES), and one page past a deferred last
+      # page; a record read twice overwrites its own file.
+      #
+      def fetch_remaining_pages(first_page) # rubocop:disable Metrics/MethodLength
+        total_pages = last = last_page(first_page)
+        deferred = []
+        page = 2
+        while page <= last
+          check_extra_pages page, total_pages
+          size = fetch_next_page(page, deferred)
+          break if size&.zero?
+
+          last = page + 1 if page == last && read_on?(size, page, total_pages)
+          page += 1
+        end
+        fetch_deferred_pages(deferred, total_pages)
+      end
+
+      def last_page(first_page)
         total = first_page.first ? first_page.first["total_count"].to_i : 0
-        total_pages = (total / PAGE_SIZE.to_f).ceil
-        (2..total_pages).each do |page|
-          records = fetch_page(page)
-          break if records.empty?
+        last = (total / PAGE_SIZE.to_f).ceil
+        first_page.size >= PAGE_SIZE ? [last, 2].max : last
+      end
+
+      #
+      # Whether to read the page after the last one: after a full page, or
+      # after a deferred page inside the total_count range. A deferred page
+      # past that range does not extend it, so a server that answers bad
+      # bodies past the end cannot keep the crawl going.
+      #
+      def read_on?(size, page, total_pages)
+        size ? size >= PAGE_SIZE : page <= total_pages
+      end
+
+      def check_extra_pages(page, total_pages)
+        return if page <= total_pages + MAX_EXTRA_PAGES
+
+        raise BadPage, "ETSI page #{page} is still full #{MAX_EXTRA_PAGES} " \
+                       "pages past total_count (#{total_pages} pages)."
+      end
+
+      #
+      # @return [Integer, nil] the number of records on the page; nil for a
+      #   deferred page
+      #
+      def fetch_next_page(page, deferred)
+        records = fetch_page(page)
+        process_records(records)
+        records.size
+      rescue BadPage => e
+        Util.warn "#{e.message} Fetching it again after the last page."
+        deferred << page
+        nil
+      end
 
-          process_records(records)
+      def fetch_deferred_pages(pages, total_pages)
+        return if pages.empty?
+
+        sleep DEFERRED_DELAY
+        failed = pages.filter_map do |page|
+          refetch_page page, total_pages
+        rescue BadPage => e
+          e.message
         end
+        raise BadPage, failed.join(" ") if failed.any?
+      end
+
+      # @return [String, nil] the failure message, nil on success
+      def refetch_page(page, total_pages)
+        records = fetch_page(page)
+        if records.empty? && page <= total_pages
+          return "ETSI page #{page} is empty on the re-fetch."
+        end
+
+        process_records records
+        nil
       end
 
       def fetch_page(page)
         date = Time.now.to_date + 1
         timestamp = (Time.now.to_f * 1000).to_i
         url = format(SOURCEURL, page: page, date: date, timestamp: timestamp)
-        JSON.parse(fetch_with_retry(url))
+        fetch_with_retry(url) { |body| parse_page(page, body) }
+      end
+
+      #
+      # ETSI's data.php sometimes answers with PHP's `Array` text in place of
+      # JSON (relaton-data-etsi crawl 36761804954); a re-fetch gets JSON.
+      #
+      # @raise [BadPage] when the body is not a JSON array
+      #
+      def parse_page(page, body)
+        records = JSON.parse(body)
+        return records if records.is_a?(Array)
+
+        raise BadPage, bad_page_message(page, body)
+      rescue JSON::ParserError
+        raise BadPage, bad_page_message(page, body)
+      end
+
+      def bad_page_message(page, body)
+        "ETSI page #{page} is not a JSON array: #{body.to_s[0, 200].inspect}."
       end
 
       def process_records(records)
@@ -92,16 +187,30 @@ module Relaton
         "Published"
       end
 
+      # A page body that is not a JSON array. See #parse_page.
+      class BadPage < StandardError; end
+
+      # Seconds to wait before a bad page is fetched again after the last page.
+      DEFERRED_DELAY = 60
+
+      # Full pages read past page 1's total_count before the crawl fails.
+      MAX_EXTRA_PAGES = 10
+
       NETWORK_ERRORS = [
         Mechanize::Error, Net::OpenTimeout, Net::ReadTimeout,
         SocketError, Errno::ECONNRESET
       ].freeze
 
+      #
+      # @yield [String] the body; the block's result is returned, and a BadPage
+      #   it raises is retried like a network error
+      #
       def fetch_with_retry(url, retries: 3, delay: 2) # rubocop:disable Metrics/MethodLength
         attempt = 0
         begin
-          Mechanize.new.get(url).body
-        rescue *NETWORK_ERRORS => e
+          body = Mechanize.new.get(url).body
+          block_given? ? yield(body) : body
+        rescue *NETWORK_ERRORS, BadPage => e
           attempt += 1
           if attempt <= retries
             Util.info "Fetch failed (#{e.message}), " \

data/lib/relaton/iec/processor.rb

@@ -7,7 +7,7 @@ module Relaton
         @short = :relaton_iec
         @prefix = "IEC"
         @pubid_flavor = :Iec # global prefixes sourced from Pubid::Iec.prefixes
-        @defaultprefix = %r{^(IEC\s|CISPR\s|IEV($|\s))}
+        @defaultprefix = %r{^(IEC\s|CISPR\s|IEV($|\s)|CEI\s)} # CEI: IEC's French spelling (relaton#243)
         @idtype = "IEC"
         @datasets = %w[iec-harmonized-all iec-harmonized-latest]
       end

data/lib/relaton/index/config.rb

@@ -4,7 +4,8 @@ module Relaton
     # Configuration class for Relaton::Index
     #
     class Config
-      attr_reader :storage, :storage_dir, :filename
+      attr_reader :storage, :storage_dir, :filename, :build_sidecar_in_child,
+                  :sqlite_index
 
       #
       # Set default values
@@ -13,6 +14,23 @@ module Relaton
         @storage = FileStorage
         @storage_dir = Dir.home
         @filename = "index.yaml"
+        @build_sidecar_in_child = true
+        @sqlite_index = true
+      end
+
+      # Build the sidecar in a forked child when available, so the one-time
+      # full materialization's memory dies with the child (relaton#242).
+      # Set to false to force an in-process build.
+      def build_sidecar_in_child=(flag)
+        @build_sidecar_in_child = flag
+      end
+
+      # Materialize a downloaded index into SQLite so a search answers from
+      # a bucket query and the process never holds the whole index
+      # (relaton#242 phase 2). Set to false to keep the raw-row sidecar path
+      # only.
+      def sqlite_index=(flag)
+        @sqlite_index = flag
       end
 
       #

data/lib/relaton/index/file_io.rb

@@ -11,6 +11,10 @@ module Relaton
       # wrong-structure handling (re-download, or stop and log).
       class InvalidIndexError < StandardError; end
 
+      # Bump when the sidecar payload shape changes, so an older sidecar is
+      # discarded and rebuilt instead of misread.
+      SIDECAR_VERSION = 2 # v2: the payload carries the yaml byte-size for freshness
+
       attr_reader :url, :pubid_class
       attr_accessor :sorted
 
@@ -271,6 +275,8 @@ module Relaton
           end
         end.to_yaml
         Index.config.storage.write file, yaml
+        delete_sidecar
+        delete_sqlite
       end
 
       def sort_structured_index(index)
@@ -288,11 +294,339 @@ module Relaton
       #
       def remove
         Index.config.storage.remove file
+        delete_sidecar
+        delete_sqlite
         []
       end
 
+      #
+      # Raw-row read for the lazy search path (relaton#242 stopgap): returns
+      # precomputed root-number sort keys and the rows as plain hashes — no
+      # pubid objects. A Marshal sidecar next to the yaml holds them so
+      # repeat loads skip the YAML parse and the one-time full
+      # materialization; the sidecar is rebuilt whenever the yaml is newer.
+      #
+      # @return [Array<Array<String>, Array<Hash>] sort keys and raw rows
+      #
+      def read_raw
+        case url
+        when String
+          with_file_lock do
+            check_file ? read_raw_file : fetch_raw_and_save
+          end
+        else
+          read_raw_file || [[], []]
+        end
+      end
+
+      # ── SQLite backend (relaton#242 phase 2) ──
+      #
+      # The downloaded index is materialized once into a SQLite database
+      # keyed by the narrowing number. A search answers from a bucket query,
+      # so the process holds O(bucket) rows instead of the whole index. The
+      # build is pure hash transforms (no pubid objects), run in a forked
+      # child so even the one-time YAML parse never lands in this process.
+
+      def sqlite_ready?
+        url.is_a?(String) && Index.config.sqlite_index != false &&
+          File.file?(sqlite_db_path) && sqlite_fresh?
+      end
+
+      # Ensure the db exists and is current (24 h TTL, schema-versioned).
+      def ensure_sqlite
+        return false unless url.is_a?(String)
+        return true if sqlite_ready?
+
+        with_file_lock do
+          build_sqlite unless sqlite_ready?
+        end
+        sqlite_ready?
+      end
+
+      def sqlite_bucket(number)
+        sqlite_backend.bucket(number)
+      end
+
+      def sqlite_count
+        sqlite_backend.count
+      end
+
+      def close_sqlite
+        @sqlite_backend&.close
+        @sqlite_backend = nil
+      end
+
+      def delete_sqlite
+        close_sqlite
+        File.delete(sqlite_db_path) if File.file?(sqlite_db_path)
+      rescue Errno::EACCES
+        nil
+      end
+
+      def sqlite_db_path
+        "#{path_to_local_file}.db"
+      end
+
+      private
+
+      def sqlite_backend
+        @sqlite_backend ||= SqliteBackend.new(sqlite_db_path, pubid_class: @pubid_class)
+      end
+
+      def sqlite_fresh?
+        ctime = Index.config.storage.ctime(sqlite_db_path)
+        backend = sqlite_backend
+        ctime && ctime > Time.now - 86400 && !backend.stale_schema?
+      rescue SQLite3::SQLException
+        false
+      ensure
+        backend&.close
+        @sqlite_backend = nil
+      end
+
+      # The build downloads the published zip, converts its rows to
+      # `[number, sort_key, id_json, file]` tuples by hash transforms alone,
+      # and writes the db — in a forked child where the YAML parse's memory
+      # dies with the process.
+      def build_sqlite
+        # The whole build — download, YAML parse, db write — runs in a
+        # forked child where available: the parse's ~300 MB (per flavor,
+        # larger for IETF) dies with the child, and the serving process
+        # never allocates it.
+        if Process.respond_to?(:fork) && !ENV["RELATON_NO_FORK"]
+          pid = Process.fork do
+            tuples = download_tuples
+            write_sqlite(tuples) if tuples
+            exit!(0)
+          end
+          Process.wait(pid)
+          true
+        else
+          tuples = download_tuples
+          return false unless tuples
+
+          write_sqlite(tuples)
+        end
+      end
+
+      def write_sqlite(tuples)
+        File.dirname(sqlite_db_path).then { |d| require "fileutils"; FileUtils.mkdir_p(d) }
+        backend = SqliteBackend.new(sqlite_db_path, pubid_class: @pubid_class)
+        backend.build(tuples.each)
+        backend.close
+      end
+
+      # Enumerate `[number, sort_key, id_hash, file]` from the published zip.
+      # The root number — the base document's, walking `base` nesting — is
+      # computed on the raw hash; no pubid object is ever built.
+      def download_tuples
+        body = Net::HTTP.get(URI.parse(url))
+        # rubyzip may mutate the buffer it is handed; a StringIO over a copy
+        # keeps the response body intact for any later reader of it.
+        yaml = nil
+        Zip::File.open_buffer(StringIO.new(body.dup)) do |zip|
+          stream = zip.entries.first.get_input_stream
+          yaml = stream.read
+        end
+        yaml = yaml.read unless yaml.is_a?(String)
+        Util.info "Downloaded index from `#{url}` for sqlite materialization", progname
+        raw = YAML.safe_load(yaml, permitted_classes: [Symbol])
+        return nil unless check_format(raw)
+
+        Enumerator.new do |y|
+          raw.each do |r|
+            number = raw_root_number(r[:id]).to_s
+            y << [number, number, r[:id], r[:file]]
+          end
+        end
+      rescue Psych::SyntaxError, SocketError, OpenURI::HTTPError, Errno::ECONNRESET,
+             OpenSSL::SSL::SSLError => e
+        Util.info "SQLite index build failed (#{e.message}); falling back", progname
+        nil
+      end
+
+      # `#root` as a hash walk: a supplement nests its origin under `base`,
+      # and the flattened to_hash puts the supplement's own components first.
+      # Accepts string and symbol keys — a YAML round-trip with permitted
+      # Symbol may give either.
+      def raw_root_number(id_hash)
+        return nil unless id_hash.is_a?(Hash)
+
+        base = id_hash["base"] || id_hash[:base]
+        return raw_root_number(base) if base
+
+        id_hash["number"] || id_hash[:number]
+      end
+
+      public
+
+      # Deserialize raw rows into pubid rows. Only the caller knows which
+      # slice it needs, so materialization happens here rather than at load.
+      #
+      # @param [Array<Hash>] rows raw rows
+      # @return [Array<Hash>] rows with deserialized ids
+      def materialize(rows)
+        return rows unless @pubid_class
+
+        rows.map { |r| { id: deserialize_id(r[:id]), file: r[:file] } }
+      end
+
+      # Save raw rows (possibly alongside the keys they were loaded with);
+      # sorts by the same root-number key the object path sorts by.
+      #
+      # @param [Array<String>] keys sort keys, parallel to rows
+      # @param [Array<Hash>] rows raw rows
+      # @return [void]
+      def save_raw(keys, rows)
+        ordered = @pubid_class ? keys.zip(rows).sort_by { |k, _| k.to_s }
+                                     .map { |_, r| r } : rows
+        yaml = ordered.map do |item|
+          { id: item[:id], file: item[:file] }
+        end.to_yaml
+        Index.config.storage.write file, yaml
+        delete_sidecar
+      end
+
       private
 
+      def sidecar_file
+        "#{file}.ms"
+      end
+
+      def delete_sidecar
+        File.delete(sidecar_file) if File.file?(sidecar_file)
+      rescue Errno::EACCES
+        nil
+      end
+
+      def read_raw_file
+        # The sidecar is authoritative while it describes the yaml byte-for-byte
+        # (same size) — the yaml is not even parsed until the sidecar is stale
+        # or unreadable.
+        if sidecar_fresh?
+          loaded = sidecar
+          return loaded if loaded
+        end
+
+        yaml = Index.config.storage.read(file)
+        return unless yaml
+
+        begin
+          raw = YAML.safe_load(yaml, permitted_classes: [Symbol])
+        rescue Psych::SyntaxError
+          warn_local_index_error("YAML parsing error when reading")
+          delete_sidecar
+          return [[], []]
+        end
+        build_raw(raw)
+      end
+
+      # Size, not mtime: NTFS timestamp coarseness makes a rewritten yaml
+      # carry the sidecar's own mtime, so "yaml is newer" misses the rebuild.
+      def sidecar_fresh?
+        File.file?(sidecar_file) && File.file?(file) &&
+          File.size(file) == sidecar_yaml_size
+      end
+
+      def sidecar_yaml_size
+        Marshal.load(File.binread(sidecar_file))[4]
+      rescue TypeError, ArgumentError, EOFError
+        nil
+      end
+
+      def sidecar
+        version, sorted, keys, rows = Marshal.load(File.binread(sidecar_file))
+        return unless version == SIDECAR_VERSION
+
+        @sorted = sorted
+        [keys, rows]
+      rescue TypeError, ArgumentError, EOFError
+        nil
+      end
+
+      def build_raw(raw)
+        unless check_format(raw)
+          warn_local_index_error("Wrong structure of")
+          return [[], []]
+        end
+
+        return build_raw_in_child(raw) if build_sidecar_in_child?
+
+        build_raw_in_process(raw)
+      rescue InvalidIndexError
+        warn_local_index_error("Wrong structure of")
+        [[], []]
+      end
+
+      # The one-time key build materializes every row (~675 MB on the
+      # 79,993-row ISO index). Freed pages are often not returned to the OS,
+      # so an in-process build leaves the parent's RSS high — and containers
+      # OOM on RSS (relaton#242). Where fork exists, build the sidecar in a
+      # child: the graph dies with it and the parent never spikes.
+      def build_sidecar_in_child?
+        Process.respond_to?(:fork) && Index.config.build_sidecar_in_child != false
+      end
+
+      def build_raw_in_child(raw)
+        pid = Process.fork do
+          build_raw_in_process(raw)
+          exit!(0)
+        end
+        Process.wait(pid)
+        loaded = sidecar
+        return build_raw_in_process(raw) unless loaded # child failed
+
+        loaded
+      rescue Errno::ENOMEM, SystemCallError
+        build_raw_in_process(raw)
+      end
+
+      def build_raw_in_process(raw)
+        objects = deserialize_pubid(raw)
+        keys = objects.map { |r| r[:id].root.number.to_s }
+        rows = objects.map { |r| { id: raw_id(r), file: r[:file] } }
+        write_sidecar(keys, rows)
+        [keys, rows]
+      rescue InvalidIndexError
+        warn_local_index_error("Wrong structure of")
+        [[], []]
+      end
+
+      def raw_id(row)
+        id = row[:id]
+        id.respond_to?(:to_hash) ? id.to_hash : id
+      end
+
+      def write_sidecar(keys, rows)
+        File.binwrite(sidecar_file,
+                      Marshal.dump([SIDECAR_VERSION, @sorted, keys, rows,
+                                    File.size(file)]))
+      rescue Errno::EACCES, Errno::ENOENT, Errno::EROFS
+        nil # the sidecar is an optimization; a read-only dir still works
+      end
+
+      def fetch_raw_and_save
+        uri = URI.parse(url)
+        body = Net::HTTP.get(uri)
+        yaml = nil
+        Zip::File.open_buffer(body) do |zip|
+          yaml = zip.entries.first.get_input_stream.read
+        end
+        Util.info "Downloaded index from `#{url}`", progname
+        raw = YAML.safe_load(yaml, permitted_classes: [Symbol])
+        if check_format(raw)
+          save raw
+          raw = nil # release the parsed copy; read_raw_file loads the sidecar's
+          read_raw_file
+        else
+          warn_remote_index_error "Wrong structure of"
+          [[], []]
+        end
+      rescue Psych::SyntaxError
+        warn_remote_index_error "YAML parsing error when reading"
+        [[], []]
+      end
+
       def with_file_lock(&)
         @@file_locks_mutex.synchronize do
           @@file_locks[file] ||= Mutex.new

data/lib/relaton/index/sqlite_backend.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+
+module Relaton
+  module Index
+    #
+    # The SQLite materialization of one flavor index (relaton#242 phase 2).
+    #
+    # A downloaded index zip is built once into a local database whose rows
+    # are `(number, sort_key, id_json, file)` with an index on `number` — the
+    # narrowing key. A search materializes only its bucket; the whole index
+    # is never resident. WAL journaling lets one build and concurrent readers
+    # share the file across processes.
+    #
+    class SqliteBackend
+      SCHEMA_VERSION = "1"
+
+      def initialize(db_path, pubid_class: nil)
+        @db_path = db_path
+        @pubid_class = pubid_class
+        @db = SQLite3::Database.new(db_path)
+        @db.busy_timeout = 30_000
+        @db.results_as_hash = false
+        @db.execute("PRAGMA journal_mode=WAL")
+        @db.execute("PRAGMA synchronous=NORMAL")
+        create_schema
+      end
+
+      # Build from an enumerator of `[number, sort_key, id_hash, file]`
+      # tuples. Callers hand the child process's enumerator here; the tuples
+      # are written inside one transaction.
+      def build(rows_enum)
+        @db.transaction do
+          @db.execute("DELETE FROM index_rows")
+          @db.prepare("INSERT INTO index_rows (number, sort_key, id_json, file) VALUES (?, ?, ?, ?)") do |stmt|
+            rows_enum.each { |(number, sort_key, id_hash, file)| stmt.execute(number, sort_key, JSON.generate(id_hash), file) }
+          end
+          meta_set("schema_version", SCHEMA_VERSION)
+        end
+      end
+
+      # The bucket for a narrowing key: raw rows, ordered as written.
+      def bucket(number)
+        rows = []
+        @db.execute("SELECT id_json, file FROM index_rows WHERE number = ? ORDER BY rowid", [number]) do |row|
+          rows << { id: JSON.parse(row[0]), file: row[1] }
+        end
+        rows
+      end
+
+      def each_row
+        @db.execute("SELECT id_json, file FROM index_rows ORDER BY rowid") do |row|
+          yield({ id: JSON.parse(row[0]), file: row[1] })
+        end
+      end
+
+      def count
+        @db.get_first_value("SELECT COUNT(*) FROM index_rows").to_i
+      end
+
+      def stale_schema?
+        meta_get("schema_version") != SCHEMA_VERSION
+      end
+
+      def close
+        @db.close
+      end
+
+      private
+
+      def create_schema
+        @db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS index_rows (
+            number TEXT NOT NULL,
+            sort_key TEXT NOT NULL,
+            id_json TEXT NOT NULL,
+            file TEXT NOT NULL
+          )
+        SQL
+        @db.execute("CREATE INDEX IF NOT EXISTS idx_index_rows_number ON index_rows (number)")
+        @db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS index_meta (
+            key TEXT PRIMARY KEY,
+            value TEXT NOT NULL
+          )
+        SQL
+      end
+
+      def meta_set(key, value)
+        @db.execute("INSERT OR REPLACE INTO index_meta (key, value) VALUES (?, ?)", [key, value])
+      end
+
+      def meta_get(key)
+        @db.get_first_value("SELECT value FROM index_meta WHERE key = ?", [key])
+      end
+    end
+  end
+end

data/lib/relaton/index/type.rb

@@ -30,7 +30,24 @@ module Relaton
       def index
         return @source.whole_index if @source
 
-        @index ||= @file_io.read
+        @index ||= @file_io.materialize(raw_index[1])
+      end
+
+      # Raw rows plus their precomputed root-number sort keys (relaton#242
+      # stopgap). The lazy search path materializes pubid objects only for
+      # the bucket it narrows to, so a process that answers one lookup holds
+      # raw hashes instead of the whole pubid graph. A caller-seeded `@index`
+      # (a spec fixture, a preloaded pool entry) is the source of truth: the
+      # raw side derives from it, in its order, so a seeder that sorts by the
+      # narrowing key keeps the bsearch valid.
+      def raw_index
+        @raw_index ||= if @index
+                          keys = @index.map { |r| narrowing_key(r[:id]) }
+                          rows = @index.map { |r| { id: raw_row_id(r[:id]), file: r[:file] } }
+                          [keys, rows]
+                        else
+                          @file_io.read_raw
+                        end
       end
 
       #
@@ -59,14 +76,23 @@ module Relaton
       # @return [void]
       #
       def add_or_update(id, file)
-        key = id.to_s
-        item = id_lookup[key]
-        if item
-          item[:file] = file
+        # A seeded or materialized @index (spec fixtures, preloaded pool
+        # entries) and pubid-less indexes run the legacy object path
+        # bit-for-bit: their consumers match rows by object identity
+        # semantics (`==`, `exclude`, to_s dedup) that raw rows cannot
+        # reproduce. Only a pubid-backed FileIO-loaded index — the
+        # relaton#242 memory case — adds through raw rows.
+        return legacy_add(id, file) if @index || !@file_io.pubid_class
+
+        raw_hash = raw_row_id(id)
+        keys, rows = raw_index
+        pos = raw_position(raw_hash)
+        if pos
+          rows[pos] = { id: raw_hash, file: file }
         else
-          new_item = { id: id, file: file }
-          index << new_item
-          id_lookup[key] = new_item
+          rows << { id: raw_hash, file: file }
+          keys << narrowing_key(id)
+          @raw_lookup[raw_hash] = rows.size - 1
           @file_io.sorted = false
         end
       end
@@ -110,7 +136,12 @@ module Relaton
       # @return [void]
       #
       def save
-        @file_io.save(@index || [])
+        if @index
+          @file_io.save(@index)
+        else
+          keys, rows = raw_index
+          @file_io.save_raw(keys, rows)
+        end
       end
 
       #
@@ -125,6 +156,8 @@ module Relaton
         @source = new_source
         @index = nil
         @id_lookup = nil
+        @raw_index = nil
+        @raw_lookup = nil
       end
 
       #
@@ -135,6 +168,8 @@ module Relaton
       def remove_all
         @index = []
         @id_lookup = nil
+        @raw_index = [[], []]
+        @raw_lookup = {}
         @file_io.sorted = true
       end
 
@@ -146,6 +181,44 @@ module Relaton
         end
       end
 
+      # Position of a raw row by its id hash — the raw-side equivalent of
+      # `id_lookup`, built once, so a crawl's add_or_update stays O(1) per row.
+      def raw_position(raw_hash)
+        @raw_lookup ||= raw_index[1].each_with_object({}) do |row, h|
+          h[row[:id]] = h.key?(row[:id]) ? h[row[:id]] : rows_index_of(h, row)
+        end
+        @raw_lookup[raw_hash]
+      end
+
+      def rows_index_of(lookup, row)
+        raw_index[1].index { |r| r[:id].equal?(row[:id]) || r[:id] == row[:id] }
+      end
+      # The pre-#242 add path, kept verbatim for seeded/materialized and
+      # pubid-less indexes.
+      def legacy_add(id, file)
+        key = id.to_s
+        item = id_lookup[key]
+        if item
+          item[:file] = file
+        else
+          new_item = { id: id, file: file }
+          index << new_item
+          id_lookup[key] = new_item
+          @file_io.sorted = false
+        end
+      end
+
+      # Same key expression the FileIO sidecar computes at build time; a
+      # String id (a no-pubid_class index) has no `.root`, and never narrows.
+      def narrowing_key(id)
+        id.respond_to?(:root) && id.root.respond_to?(:number) ? id.root.number.to_s : ""
+      end
+
+      def raw_row_id(id)
+        pc = @file_io.pubid_class
+        pc && id.is_a?(pc) ? id.to_hash : id
+      end
+
       def new_source
         ShardSource.new(@dir, @pages_url, @pubid_class) if @pages_url
       end
@@ -155,13 +228,26 @@ module Relaton
         # never refreshed, so the shards stay the fresher answer.
         return @source.rows(id) if @source && id && !id.is_a?(String)
 
-        # index needs to be created to check if sorted
-        idx = index
-        if @file_io.sorted && id && !id.is_a?(String)
-          candidates_by_number(id)
-        else
-          idx
+        # The SQLite backend answers a parsed query with a bucket query:
+        # O(bucket) rows ever resident, no sidecar load at all. Falls back
+        # to the raw-row path when the backend cannot build or is disabled.
+        if id && !id.is_a?(String) && @file_io.pubid_class
+          begin
+            return sqlite_candidates(id) if @file_io.ensure_sqlite
+          rescue SQLite3::SQLException
+            nil # fall back to the sidecar path below
+          end
+
+          raw_index
+          return candidates_by_number(id) if @file_io.sorted
         end
+        index
+      end
+
+      def sqlite_candidates(id)
+        target = id.root.number.to_s
+        rows = @file_io.sqlite_bucket(target)
+        @file_io.materialize(rows)
       end
 
       # Narrowing key: the base *document's* number as a string. `#root` walks a
@@ -169,26 +255,24 @@ module Relaton
       # returns self for a base document), so a document and all its wrappers
       # share one key and cluster together. `.to_s` because the key is compared
       # as a string (a pubid number Component is not `<`/`>`-comparable), and
-      # FileIO sorts the index by this exact same key so bsearch stays valid.
+      # FileIO computes this exact same key once at sidecar-build time so the
+      # bsearch over the keys array stays valid.
       def candidates_by_number(id)
         target = id.root.number.to_s
+        keys, rows = raw_index
         left = bsearch_left(target)
         return [] unless left
 
         right = bsearch_right(target)
-        index[left...right]
+        @file_io.materialize(rows[left...right])
       end
 
       def bsearch_left(target)
-        index.bsearch_index do |item|
-          item[:id].root.number.to_s >= target
-        end
+        raw_index[0].bsearch_index { |k| k >= target }
       end
 
       def bsearch_right(target)
-        index.bsearch_index do |item|
-          item[:id].root.number.to_s > target
-        end || index.size
+        raw_index[0].bsearch_index { |k| k > target } || raw_index[0].size
       end
 
       # The query is the reference, so two identifiers take the subset match.

data/lib/relaton/index.rb

@@ -2,6 +2,7 @@
 
 require "yaml"
 require "zip"
+require "open-uri"
 require "relaton/logger"
 
 require_relative "version"
@@ -10,9 +11,10 @@ require_relative "index/file_storage"
 require_relative "index/config"
 require_relative "index/util"
 require_relative "index/pool"
-require_relative "index/type"
 require_relative "index/file_io"
 require_relative "index/shard_source"
+require_relative "index/sqlite_backend"
+require_relative "index/type"
 
 module Relaton
   module Index

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "3.0.0.pre.alpha.6".freeze
+  VERSION = "3.0.0.pre.alpha.8".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre.alpha.6
+  version: 3.0.0.pre.alpha.8
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-02 00:00:00.000000000 Z
+date: 2026-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement
@@ -933,6 +947,7 @@ files:
 - lib/relaton/index/file_storage.rb
 - lib/relaton/index/pool.rb
 - lib/relaton/index/shard_source.rb
+- lib/relaton/index/sqlite_backend.rb
 - lib/relaton/index/type.rb
 - lib/relaton/index/util.rb
 - lib/relaton/isbn.rb