relaton 3.0.0.pre.alpha.7 → 3.0.0.pre.alpha.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21f068205ef6a80e718953c7989c7a51219a9f5f6e25a4fecc67ac93d9fa4da5
-  data.tar.gz: 7c438465496dc3b95370bc6990420073c63f41d5edba70022a71b39bd45f343f
+  metadata.gz: 2c1a04ce086b4f13d9ec8c448b4254cd3f4bc0399d0d1b4892e2201f45717f4c
+  data.tar.gz: f5a8286c7313b4f47a131290923ad2d23082cd51fb492f20555d859f38375cfd
 SHA512:
-  metadata.gz: af94d0ffa5ae9cbeefecf98d9aad8a04a7ea74814b92e470b6235b2bb1c2f19593db50e0f919b2ce3bbf1bdf44b8e637fa20000e6d59bc400736159fe0f9a2d1
-  data.tar.gz: 833bbe454eaa888a145fb9dcd5daff7d8bc8cf2e03f603c2100b0d19d981531a5b4bfce87b77bc00f3b5a5684904473f58656ebf820b2c7272b606f5c26afddc
+  metadata.gz: 63355667dfc69e951fd03842f8b3769dae41ffce420b4deb21349dac2a035db489b17c808d9c6b8c478f324c15715c1d6cbc7d00422f47f46d649443ea9f3a63
+  data.tar.gz: bbe9885dd3811d1004a2048dd0dd88846ded2075ca21f26fe7b32f5286d63f5a014b83f0fd681bd9db61400f54bc3fabc9ec4998d791bcac578be07cfe290645

data/lib/relaton/bib/model/ics.rb

@@ -29,8 +29,7 @@ module Relaton
         return unless val.is_a?(String) && !val.empty?
         return if @text.is_a?(String) && !@text.empty?
 
-        description = Isoics.fetch(val)&.description
-        self.text = description if description
+        populate_text_from_isoics
       end
 
       # When the deserializer reaches the end of the XML element and
@@ -38,9 +37,44 @@ module Relaton
       # to mark the attribute as default-valued (suppressing serialization).
       # Refuse that mark if we've already populated text from Isoics so the
       # value survives round-trip. See #112.
+      # lutaml-model 0.8.92 (lutaml/lutaml-model#922): assignments made
+      # inside `code=` while from_xml is still mid-element do not register
+      # as explicit values. Populate once deserialization has finished.
+      def self.from_xml(node)
+        ics = super
+        # lutaml-model 0.8.92 (#922): a value assigned through a custom
+        # writer's `super` mid-deserialization is left default-suppressed.
+        # Register code as explicitly set, then populate text.
+        ics.value_set_for(:code) if ics.code.is_a?(String) && !ics.code.empty?
+        ics.populate_text_from_isoics
+        ics
+      end
+
+      def populate_text_from_isoics
+        code.is_a?(String) && !code.empty? or return
+        @text.is_a?(String) && !@text.empty? and return
+
+        description = Isoics.fetch(code)&.description
+        self.text = description if description
+      end
+
       def using_default_for(attribute_name)
-        return if attribute_name == :text &&
-          @text.is_a?(String) && !@text.empty?
+        # lutaml-model 0.8.92 (#922) leaves values assigned through a
+        # custom writer's `super` default-suppressed. code, when present,
+        # is always explicit; text absent from the XML is filled from
+        # Isoics here — the element end — and emitted.
+        if attribute_name == :code
+          return value_set_for(:code) if @code.is_a?(String) && !@code.empty?
+
+          return super
+        end
+
+        return if attribute_name == :text && @text.is_a?(String) && !@text.empty?
+
+        if attribute_name == :text
+          populate_text_from_isoics
+          return value_set_for(:text) unless @text.nil? || @text.empty?
+        end
 
         super
       end

data/lib/relaton/index/config.rb

@@ -4,7 +4,8 @@ module Relaton
     # Configuration class for Relaton::Index
     #
     class Config
-      attr_reader :storage, :storage_dir, :filename, :build_sidecar_in_child
+      attr_reader :storage, :storage_dir, :filename, :build_sidecar_in_child,
+                  :sqlite_index
 
       #
       # Set default values
@@ -14,6 +15,7 @@ module Relaton
         @storage_dir = Dir.home
         @filename = "index.yaml"
         @build_sidecar_in_child = true
+        @sqlite_index = true
       end
 
       # Build the sidecar in a forked child when available, so the one-time
@@ -23,6 +25,14 @@ module Relaton
         @build_sidecar_in_child = flag
       end
 
+      # Materialize a downloaded index into SQLite so a search answers from
+      # a bucket query and the process never holds the whole index
+      # (relaton#242 phase 2). Set to false to keep the raw-row sidecar path
+      # only.
+      def sqlite_index=(flag)
+        @sqlite_index = flag
+      end
+
       #
       # Set storage
       #

data/lib/relaton/index/file_io.rb

@@ -275,6 +275,8 @@ module Relaton
           end
         end.to_yaml
         Index.config.storage.write file, yaml
+        delete_sidecar
+        delete_sqlite
       end
 
       def sort_structured_index(index)
@@ -293,14 +295,15 @@ module Relaton
       def remove
         Index.config.storage.remove file
         delete_sidecar
+        delete_sqlite
         []
       end
 
       #
       # Raw-row read for the lazy search path (relaton#242 stopgap): returns
       # precomputed root-number sort keys and the rows as plain hashes — no
-      # pubid objects. A Marshal sidecar next to the yaml holds the keys+rows
-      # so repeat loads skip the YAML parse and the one-time full
+      # pubid objects. A Marshal sidecar next to the yaml holds them so
+      # repeat loads skip the YAML parse and the one-time full
       # materialization; the sidecar is rebuilt whenever the yaml is newer.
       #
       # @return [Array<Array<String>, Array<Hash>] sort keys and raw rows
@@ -316,6 +319,147 @@ module Relaton
         end
       end
 
+      # ── SQLite backend (relaton#242 phase 2) ──
+      #
+      # The downloaded index is materialized once into a SQLite database
+      # keyed by the narrowing number. A search answers from a bucket query,
+      # so the process holds O(bucket) rows instead of the whole index. The
+      # build is pure hash transforms (no pubid objects), run in a forked
+      # child so even the one-time YAML parse never lands in this process.
+
+      def sqlite_ready?
+        url.is_a?(String) && Index.config.sqlite_index != false &&
+          File.file?(sqlite_db_path) && sqlite_fresh?
+      end
+
+      # Ensure the db exists and is current (24 h TTL, schema-versioned).
+      def ensure_sqlite
+        return false unless url.is_a?(String)
+        return true if sqlite_ready?
+
+        with_file_lock do
+          build_sqlite unless sqlite_ready?
+        end
+        sqlite_ready?
+      end
+
+      def sqlite_bucket(number)
+        sqlite_backend.bucket(number)
+      end
+
+      def sqlite_count
+        sqlite_backend.count
+      end
+
+      def close_sqlite
+        @sqlite_backend&.close
+        @sqlite_backend = nil
+      end
+
+      def delete_sqlite
+        close_sqlite
+        File.delete(sqlite_db_path) if File.file?(sqlite_db_path)
+      rescue Errno::EACCES
+        nil
+      end
+
+      def sqlite_db_path
+        "#{path_to_local_file}.db"
+      end
+
+      private
+
+      def sqlite_backend
+        @sqlite_backend ||= SqliteBackend.new(sqlite_db_path, pubid_class: @pubid_class)
+      end
+
+      def sqlite_fresh?
+        ctime = Index.config.storage.ctime(sqlite_db_path)
+        backend = sqlite_backend
+        ctime && ctime > Time.now - 86400 && !backend.stale_schema?
+      rescue SQLite3::SQLException
+        false
+      ensure
+        backend&.close
+        @sqlite_backend = nil
+      end
+
+      # The build downloads the published zip, converts its rows to
+      # `[number, sort_key, id_json, file]` tuples by hash transforms alone,
+      # and writes the db — in a forked child where the YAML parse's memory
+      # dies with the process.
+      def build_sqlite
+        # The whole build — download, YAML parse, db write — runs in a
+        # forked child where available: the parse's ~300 MB (per flavor,
+        # larger for IETF) dies with the child, and the serving process
+        # never allocates it.
+        if Process.respond_to?(:fork) && !ENV["RELATON_NO_FORK"]
+          pid = Process.fork do
+            tuples = download_tuples
+            write_sqlite(tuples) if tuples
+            exit!(0)
+          end
+          Process.wait(pid)
+          true
+        else
+          tuples = download_tuples
+          return false unless tuples
+
+          write_sqlite(tuples)
+        end
+      end
+
+      def write_sqlite(tuples)
+        File.dirname(sqlite_db_path).then { |d| require "fileutils"; FileUtils.mkdir_p(d) }
+        backend = SqliteBackend.new(sqlite_db_path, pubid_class: @pubid_class)
+        backend.build(tuples.each)
+        backend.close
+      end
+
+      # Enumerate `[number, sort_key, id_hash, file]` from the published zip.
+      # The root number — the base document's, walking `base` nesting — is
+      # computed on the raw hash; no pubid object is ever built.
+      def download_tuples
+        body = Net::HTTP.get(URI.parse(url))
+        # rubyzip may mutate the buffer it is handed; a StringIO over a copy
+        # keeps the response body intact for any later reader of it.
+        yaml = nil
+        Zip::File.open_buffer(StringIO.new(body.dup)) do |zip|
+          stream = zip.entries.first.get_input_stream
+          yaml = stream.read
+        end
+        yaml = yaml.read unless yaml.is_a?(String)
+        Util.info "Downloaded index from `#{url}` for sqlite materialization", progname
+        raw = YAML.safe_load(yaml, permitted_classes: [Symbol])
+        return nil unless check_format(raw)
+
+        Enumerator.new do |y|
+          raw.each do |r|
+            number = raw_root_number(r[:id]).to_s
+            y << [number, number, r[:id], r[:file]]
+          end
+        end
+      rescue Psych::SyntaxError, SocketError, OpenURI::HTTPError, Errno::ECONNRESET,
+             OpenSSL::SSL::SSLError => e
+        Util.info "SQLite index build failed (#{e.message}); falling back", progname
+        nil
+      end
+
+      # `#root` as a hash walk: a supplement nests its origin under `base`,
+      # and the flattened to_hash puts the supplement's own components first.
+      # Accepts string and symbol keys — a YAML round-trip with permitted
+      # Symbol may give either.
+      def raw_root_number(id_hash)
+        return nil unless id_hash.is_a?(Hash)
+
+        base = id_hash["base"] || id_hash[:base]
+        return raw_root_number(base) if base
+
+        id_hash["number"] || id_hash[:number]
+      end
+
+      public
+
       # Deserialize raw rows into pubid rows. Only the caller knows which
       # slice it needs, so materialization happens here rather than at load.
       #

data/lib/relaton/index/sqlite_backend.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+
+module Relaton
+  module Index
+    #
+    # The SQLite materialization of one flavor index (relaton#242 phase 2).
+    #
+    # A downloaded index zip is built once into a local database whose rows
+    # are `(number, sort_key, id_json, file)` with an index on `number` — the
+    # narrowing key. A search materializes only its bucket; the whole index
+    # is never resident. WAL journaling lets one build and concurrent readers
+    # share the file across processes.
+    #
+    class SqliteBackend
+      SCHEMA_VERSION = "1"
+
+      def initialize(db_path, pubid_class: nil)
+        @db_path = db_path
+        @pubid_class = pubid_class
+        @db = SQLite3::Database.new(db_path)
+        @db.busy_timeout = 30_000
+        @db.results_as_hash = false
+        @db.execute("PRAGMA journal_mode=WAL")
+        @db.execute("PRAGMA synchronous=NORMAL")
+        create_schema
+      end
+
+      # Build from an enumerator of `[number, sort_key, id_hash, file]`
+      # tuples. Callers hand the child process's enumerator here; the tuples
+      # are written inside one transaction.
+      def build(rows_enum)
+        @db.transaction do
+          @db.execute("DELETE FROM index_rows")
+          @db.prepare("INSERT INTO index_rows (number, sort_key, id_json, file) VALUES (?, ?, ?, ?)") do |stmt|
+            rows_enum.each { |(number, sort_key, id_hash, file)| stmt.execute(number, sort_key, JSON.generate(id_hash), file) }
+          end
+          meta_set("schema_version", SCHEMA_VERSION)
+        end
+      end
+
+      # The bucket for a narrowing key: raw rows, ordered as written.
+      def bucket(number)
+        rows = []
+        @db.execute("SELECT id_json, file FROM index_rows WHERE number = ? ORDER BY rowid", [number]) do |row|
+          rows << { id: JSON.parse(row[0]), file: row[1] }
+        end
+        rows
+      end
+
+      def each_row
+        @db.execute("SELECT id_json, file FROM index_rows ORDER BY rowid") do |row|
+          yield({ id: JSON.parse(row[0]), file: row[1] })
+        end
+      end
+
+      def count
+        @db.get_first_value("SELECT COUNT(*) FROM index_rows").to_i
+      end
+
+      def stale_schema?
+        meta_get("schema_version") != SCHEMA_VERSION
+      end
+
+      def close
+        @db.close
+      end
+
+      private
+
+      def create_schema
+        @db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS index_rows (
+            number TEXT NOT NULL,
+            sort_key TEXT NOT NULL,
+            id_json TEXT NOT NULL,
+            file TEXT NOT NULL
+          )
+        SQL
+        @db.execute("CREATE INDEX IF NOT EXISTS idx_index_rows_number ON index_rows (number)")
+        @db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS index_meta (
+            key TEXT PRIMARY KEY,
+            value TEXT NOT NULL
+          )
+        SQL
+      end
+
+      def meta_set(key, value)
+        @db.execute("INSERT OR REPLACE INTO index_meta (key, value) VALUES (?, ?)", [key, value])
+      end
+
+      def meta_get(key)
+        @db.get_first_value("SELECT value FROM index_meta WHERE key = ?", [key])
+      end
+    end
+  end
+end

data/lib/relaton/index/type.rb

@@ -228,17 +228,28 @@ module Relaton
         # never refreshed, so the shards stay the fresher answer.
         return @source.rows(id) if @source && id && !id.is_a?(String)
 
-        # The lazy path narrows on the precomputed keys and materializes only
-        # the bucket; the materialized `index` stays available to callers
-        # that need the whole graph (`#index`, a String query, a block).
-        # Load first: a pubid-backed load is what settles @file_io.sorted.
+        # The SQLite backend answers a parsed query with a bucket query:
+        # O(bucket) rows ever resident, no sidecar load at all. Falls back
+        # to the raw-row path when the backend cannot build or is disabled.
         if id && !id.is_a?(String) && @file_io.pubid_class
+          begin
+            return sqlite_candidates(id) if @file_io.ensure_sqlite
+          rescue SQLite3::SQLException
+            nil # fall back to the sidecar path below
+          end
+
           raw_index
           return candidates_by_number(id) if @file_io.sorted
         end
         index
       end
 
+      def sqlite_candidates(id)
+        target = id.root.number.to_s
+        rows = @file_io.sqlite_bucket(target)
+        @file_io.materialize(rows)
+      end
+
       # Narrowing key: the base *document's* number as a string. `#root` walks a
       # supplement/amendment/corrigendum's `.base` chain to the origin (and
       # returns self for a base document), so a document and all its wrappers

data/lib/relaton/index.rb

@@ -2,6 +2,7 @@
 
 require "yaml"
 require "zip"
+require "open-uri"
 require "relaton/logger"
 
 require_relative "version"
@@ -10,9 +11,10 @@ require_relative "index/file_storage"
 require_relative "index/config"
 require_relative "index/util"
 require_relative "index/pool"
-require_relative "index/type"
 require_relative "index/file_io"
 require_relative "index/shard_source"
+require_relative "index/sqlite_backend"
+require_relative "index/type"
 
 module Relaton
   module Index

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "3.0.0.pre.alpha.7".freeze
+  VERSION = "3.0.0.pre.alpha.9".freeze
 end

data/lib/relaton.rb

@@ -7,16 +7,16 @@
 require "relaton/version"
 require "moxml"
 
-# YAML through yeptris (Psych-compatible drop-in; stdlib Psych is kept
-# as Yeptris::Psych::ORIGINAL). Must load before any YAML/Psych use.
-# yeptris 0.5 moved the exclusive drop-in rebind to yeptris/psych/drop_in
-# — plain yeptris/psych only defines the namespace and leaves stdlib
-# Psych in place (yeptris-ruby#95, bug 3).
-# Load stdlib Psych first: the drop-in keeps it as Yeptris::Psych::ORIGINAL,
-# and a later plain require "psych" against the rebound constant is a
-# superclass mismatch (yeptris-ruby#95, bug 3 follow-on).
-require "psych"
-require "yeptris/psych/drop_in"
+# YAML through stdlib Psych. The former yeptris drop-in rebind
+# (require "yeptris/psych/drop_in") replaced ::Psych for the WHOLE
+# host process: every gem that later loaded YAML — glossarist/iev's
+# ISO 639 table, metanorma-cli's collections, anything psych-lenient
+# — silently changed engine, and yeptris 0.6.28.4 diverges from psych
+# (block-scalar headers at column 1, permitted_classes ignored; see
+# yeptris-ruby#258). A library must never rebind global constants;
+# the namespaced Yeptris::Psych API remains the opt-in path for hosts
+# that want it.
+require "yaml"
 
 module Relaton
   autoload :Logger, "relaton/logger"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: relaton
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre.alpha.7
+  version: 3.0.0.pre.alpha.9
 platform: ruby
 authors:
 - Ribose Inc.
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement
@@ -933,6 +947,7 @@ files:
 - lib/relaton/index/file_storage.rb
 - lib/relaton/index/pool.rb
 - lib/relaton/index/shard_source.rb
+- lib/relaton/index/sqlite_backend.rb
 - lib/relaton/index/type.rb
 - lib/relaton/index/util.rb
 - lib/relaton/isbn.rb