iriq 0.30.2 → 0.35.0

data/lib/iriq/storage/sqlite.rb

@@ -13,6 +13,24 @@ module Iriq
     class Sqlite
       SCHEMA_VERSION = 4
 
+      # Processes share a corpus's write lock by one rule: a writer waits up
+      # to LOCK_WAIT seconds for its turn, and none keeps the lock longer
+      # than a turn of work.
+      LOCK_WAIT = 10
+      LOCK_TURN = 1.0
+      # How long a writer that used a whole turn leaves the lock free: many
+      # of a waiter's 1ms busy retries, and a fair chance at an older iriq's
+      # 100ms ones.
+      TURN_PAUSE = 0.02
+
+      # Every table derived from the observation log.
+      VIEW_TABLES = %w[
+        host_counts path_length_counts raw_shape_counts fingerprint_counts
+        position_stats position_values position_types
+        clusters cluster_examples cluster_segments
+        cluster_params cluster_param_values cluster_param_types
+      ].freeze
+
       SCHEMA = <<~SQL.freeze
         CREATE TABLE IF NOT EXISTS meta (
           key   TEXT PRIMARY KEY,
@@ -128,6 +146,21 @@ module Iriq
       def self.open(path, classifier: SegmentClassifier::DEFAULT,
                           max_values_per_position: PositionStats::DEFAULT_MAX_VALUES)
         new(path: path, classifier: classifier, max_values_per_position: max_values_per_position).tap(&:setup!)
+      rescue SQLite3::Exception => e
+        raise corpus_error(path, e)
+      end
+
+      # A failure SQLite reports (not a database, can't open, read-only, full
+      # disk) reads like every other corpus failure: `corpus PATH: cause`. The
+      # gem appends the failing SQL to its message; the cause is the part before.
+      def self.corpus_error(path, e)
+        # Busy only ever means the wait for another process's lock ran out.
+        cause = if e.is_a?(SQLite3::BusyException)
+          "another process held the corpus lock for over #{LOCK_WAIT}s"
+        else
+          e.message.split(":\n", 2).first
+        end
+        CorpusError.new("corpus #{path}: #{cause}")
       end
 
       def initialize(path:, classifier: SegmentClassifier::DEFAULT,
@@ -136,23 +169,62 @@ module Iriq
         @classifier              = classifier
         @max_values_per_position = max_values_per_position
         @db                      = SQLite3::Database.new(path)
-        # busy_timeout MUST come first: other PRAGMAs (journal_mode in
-        # particular) can themselves block on the write lock under
-        # concurrent open, and without busy_timeout set they fail
-        # immediately with SQLITE_BUSY.
-        @db.execute("PRAGMA busy_timeout = 30000")
-        @db.execute("PRAGMA journal_mode = WAL")
+        # Before any PRAGMA: journal_mode itself can wait on a lock. SQLite's
+        # own busy_timeout backs off to 100ms between retries, which would
+        # mostly miss a TURN_PAUSE; this one retries every 1ms.
+        @db.busy_handler_timeout = LOCK_WAIT * 1000
+        enable_wal!
         @db.execute("PRAGMA synchronous = NORMAL")
+        # Up to 64MB of pages (a ceiling, not an allocation): an ingest that
+        # commits a turn at a time re-reads the pages it wrote last turn, and
+        # SQLite's default 2MB cache turns that into disk reads.
+        @db.execute("PRAGMA cache_size = -64000")
         @db.execute("PRAGMA foreign_keys = ON")
-        @in_batch = false
+        @in_batch       = false
+        @in_transaction = false
+        # Values tracked per position, remembered so each new value needn't
+        # re-count them. Exact only inside a transaction (the write lock is
+        # held) and while @counts_version still matches.
+        @value_counts   = {}
+        # PRAGMA data_version when @value_counts was last known exact; it
+        # changes only when another connection commits.
+        @counts_version = nil
+        # When the transaction in progress took the write lock, and until
+        # when a connection that used a whole turn leaves the lock free.
+        @locked_at      = nil
+        @next_turn      = nil
+        # A rebuild is in progress: TEMP tables named like the views shadow
+        # them for this connection alone, so every view statement writes the
+        # rebuild. Its own transaction touches only those tables; it ends
+        # before a log read, so the read sees the latest log, and before the
+        # write lock is taken.
+        @rebuilding     = false
+        @rebuild_txn    = false
+      end
+
+      # Checked before SCHEMA runs, so iriq never adds tables to a corpus
+      # written by a newer iriq.
+      def refuse_newer_schema!
+        has_meta = @db.get_first_value("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta'")
+        return unless has_meta
+
+        version = @db.get_first_value("SELECT value FROM meta WHERE key = 'schema_version'").to_i
+        return if version <= SCHEMA_VERSION
+
+        @db.close
+        raise CorpusError, "corpus #{@path}: schema version #{version} is newer than this iriq supports (#{SCHEMA_VERSION}); upgrade iriq"
       end
 
       def setup!
+        refuse_newer_schema!
         @db.execute_batch(SCHEMA)
         existing = @db.get_first_value("SELECT value FROM meta WHERE key = 'schema_version'")
         if existing.nil?
-          @db.execute("INSERT INTO meta (key, value) VALUES ('schema_version', ?)", SCHEMA_VERSION.to_s)
-          @db.execute("INSERT INTO meta (key, value) VALUES ('max_values_per_position', ?)",
+          # OR IGNORE: two processes can race to initialize a fresh corpus
+          # concurrently — both read schema_version as nil, and the loser's
+          # INSERT must not blow up on the PRIMARY KEY.
+          @db.execute("INSERT OR IGNORE INTO meta (key, value) VALUES ('schema_version', ?)", SCHEMA_VERSION.to_s)
+          @db.execute("INSERT OR IGNORE INTO meta (key, value) VALUES ('max_values_per_position', ?)",
                       @max_values_per_position.to_s)
         else
           @max_values_per_position = (@db.get_first_value(
@@ -167,32 +239,119 @@ module Iriq
         # no-ops — the outer batch wraps everything in one txn for speed.
         return yield(self) if @in_batch
 
-        @db.transaction
-        yield self
-        @db.commit
-      rescue
-        @db.rollback rescue nil
-        raise
+        write_transaction { yield self }
       end
 
       # Wrap many observations in a single transaction. Cuts SQLite write
-      # overhead from O(observations) fsyncs to O(1).
+      # overhead from O(observations) fsyncs to O(1). Yields whether another
+      # connection may have committed since this one's last transaction.
       def batch
-        return yield if @in_batch
+        return yield(false) if @in_batch
 
         @in_batch = true
-        @db.transaction
         begin
-          yield
-          @db.commit
-        rescue
-          @db.rollback rescue nil
-          raise
+          write_transaction { |changed| yield changed }
         ensure
           @in_batch = false
         end
       end
 
+      # IMMEDIATE takes the write lock up front. A deferred transaction that
+      # reads first can't upgrade once another process commits: SQLite
+      # reports busy without consulting busy_timeout.
+      private def write_transaction
+        if @rebuild_txn
+          # A rebuild's writes so far are its own; keep them while waiting.
+          @db.commit
+          @rebuild_txn = false
+        end
+        pause = @next_turn && (@next_turn - monotonic_now)
+        @next_turn = nil
+        sleep(pause) if pause&.positive?
+        @db.transaction(:immediate)
+        @locked_at      = monotonic_now
+        @in_transaction = true
+        # Under the write lock no one else can commit until we do, so the
+        # version read now holds for the whole transaction. An unknown last
+        # version (first transaction, or after a rollback) counts as changed.
+        version = @db.get_first_value("PRAGMA data_version")
+        changed = version != @counts_version
+        @value_counts.clear if changed
+        @counts_version = version
+        result = yield changed
+        @db.commit
+        result
+      rescue => e
+        @db.rollback rescue nil
+        @value_counts.clear
+        @counts_version = nil
+        raise corpus_error(e)
+      ensure
+        # One that used a whole turn leaves the lock free a moment.
+        @next_turn      = monotonic_now + TURN_PAUSE if @locked_at && monotonic_now - @locked_at >= LOCK_TURN
+        @locked_at      = nil
+        @in_transaction = false
+      end
+
+      # Whether the transaction in progress has held the write lock for a
+      # whole turn, so a long-running writer should commit and let others in.
+      def turn_over?
+        !@locked_at.nil? && monotonic_now - @locked_at >= LOCK_TURN
+      end
+
+      # --- Rebuilding the views ---------------------------------------------
+      #
+      # Rebuild out of sight: view writes after begin_rebuild go to fresh,
+      # empty views only this connection sees, until install_rebuild (inside a
+      # transaction) makes them the corpus's views. discard_rebuild drops a
+      # rebuild not installed, and also runs after an install whose
+      # transaction rolled back, which brings the TEMP tables back. Other
+      # connections keep reading and writing the live views meanwhile.
+
+      def begin_rebuild
+        discard_rebuild
+        VIEW_TABLES.each do |table|
+          # The live table's own definition, so the copy lines up column for
+          # column whichever iriq created the corpus.
+          sql = @db.get_first_value("SELECT sql FROM main.sqlite_master WHERE type = 'table' AND name = ?", [table])
+          @db.execute(sql.sub("CREATE TABLE", "CREATE TEMP TABLE"))
+        end
+        unless @in_transaction
+          @db.transaction
+          @rebuild_txn = true
+        end
+        @value_counts.clear
+        @rebuilding = true
+      rescue SQLite3::Exception => e
+        raise corpus_error(e)
+      end
+
+      def install_rebuild
+        VIEW_TABLES.each do |table|
+          @db.execute("DELETE FROM main.#{table}")
+          @db.execute("INSERT INTO main.#{table} SELECT * FROM temp.#{table}")
+          @db.execute("DROP TABLE temp.#{table}")
+        end
+        @rebuilding = false
+        @value_counts.clear
+      end
+
+      def discard_rebuild
+        if @rebuild_txn
+          @db.rollback rescue nil
+          @rebuild_txn = false
+        end
+        VIEW_TABLES.each { |table| @db.execute("DROP TABLE IF EXISTS temp.#{table}") rescue nil }
+        @rebuilding = false
+        @value_counts.clear
+      end
+
+      # SQLite failures inside a transaction become CorpusErrors; anything else
+      # passes through.
+      def corpus_error(e)
+        e.is_a?(SQLite3::Exception) ? Sqlite.corpus_error(@path, e) : e
+      end
+
       # Saving is automatic — incremental UPSERTs hit disk on commit. flush
       # makes that explicit; close releases the connection.
       def flush; end
@@ -258,16 +417,18 @@ module Iriq
           WHERE host = ? AND scope = ? AND locator = ? AND value = ?
         SQL
         if @db.changes.zero?
-          card = @db.get_first_value(
-            "SELECT COUNT(*) FROM position_values WHERE host = ? AND scope = ? AND locator = ?",
-            [host, scope, locator],
+          where = [host, scope, locator]
+          card = (counts_exact? && @value_counts[where]) || @db.get_first_value(
+            "SELECT COUNT(*) FROM position_values WHERE host = ? AND scope = ? AND locator = ?", where,
           )
           if card < @max_values_per_position
             @db.execute(
               "INSERT INTO position_values (host, scope, locator, value, count) VALUES (?, ?, ?, ?, 1)",
               [host, scope, locator, value],
             )
+            card += 1
           end
+          @value_counts[where] = card if counts_exact?
         end
       end
 
@@ -280,12 +441,18 @@ module Iriq
           ON CONFLICT(key) DO UPDATE SET count = count + 1
         SQL
 
-        # Examples — capped at Cluster::MAX_EXAMPLES.
+        # Examples — capped at Cluster::MAX_EXAMPLES, deduped by canonical
+        # (mirrors Cluster#add so the SQLite view matches the in-memory one).
+        canonical = identifier.canonical
         examples_count = @db.get_first_value(
           "SELECT COUNT(*) FROM cluster_examples WHERE cluster_key = ?", [key],
         )
-        if examples_count < Cluster::MAX_EXAMPLES
-          @db.execute(<<~SQL, [key, examples_count, identifier.canonical])
+        already_present = @db.get_first_value(
+          "SELECT 1 FROM cluster_examples WHERE cluster_key = ? AND canonical = ?",
+          [key, canonical],
+        )
+        if examples_count < Cluster::MAX_EXAMPLES && already_present.nil?
+          @db.execute(<<~SQL, [key, examples_count, canonical])
             INSERT INTO cluster_examples (cluster_key, position, canonical)
             VALUES (?, ?, ?)
           SQL
@@ -332,8 +499,6 @@ module Iriq
             end
           end
         end
-
-        load_cluster(key)
       end
 
       # Append a canonical IRI to the source-IRI log. Inside the same
@@ -349,6 +514,18 @@ module Iriq
         end
       end
 
+      # The observations logged after `mark` (0 for all), in id order;
+      # returns the id of the last one (`mark` when there are none).
+      def each_observed_iri_since(mark)
+        if @rebuild_txn
+          @db.commit
+          @db.transaction
+        end
+        rows = @db.execute("SELECT id, canonical FROM observed_iris WHERE id > ? ORDER BY id", [mark])
+        rows.each { |_id, canonical| yield canonical }
+        rows.empty? ? mark : rows.last[0]
+      end
+
       def observed_iri_count
         @db.get_first_value("SELECT COUNT(*) FROM observed_iris") || 0
       end
@@ -375,6 +552,7 @@ module Iriq
       # Drop every materialized view without touching the source-IRI log.
       # Corpus#reinfer calls this before replaying the log.
       def clear_materialized_views
+        @value_counts.clear
         @db.execute_batch(<<~SQL)
           DELETE FROM host_counts;
           DELETE FROM path_length_counts;
@@ -400,7 +578,7 @@ module Iriq
 
       def path_length_counts
         h = Hash.new(0)
-        @db.execute("SELECT length, count FROM path_length_counts") { |r| h[r[0]] = r[1] }
+        @db.execute("SELECT length, count FROM path_length_counts") { |r| h[r[0]] = coerce_int!(r[1], 1, "count") }
         h
       end
 
@@ -423,22 +601,23 @@ module Iriq
         return nil if total.nil?
 
         stats = PositionStats.new(max_values: @max_values_per_position)
-        stats.instance_variable_set(:@total, total)
+        stats.instance_variable_set(:@total, coerce_int!(total, 0, "total"))
 
         vc = Hash.new(0)
         @db.execute(
           "SELECT value, count FROM position_values WHERE host = ? AND scope = ? AND locator = ?",
           [host, scope, locator],
-        ) { |r| vc[r[0]] = r[1] }
+        ) { |r| vc[r[0]] = coerce_int!(r[1], 1, "count") }
         stats.instance_variable_set(:@value_counts, vc)
 
         tc = Hash.new(0)
         @db.execute(
           "SELECT type, count FROM position_types WHERE host = ? AND scope = ? AND locator = ?",
           [host, scope, locator],
-        ) { |r| tc[r[0].to_sym] = r[1] }
+        ) { |r| tc[r[0].to_sym] = coerce_int!(r[1], 1, "count") }
         stats.instance_variable_set(:@type_counts, tc)
 
+        recompute_numeric!(stats)
         stats
       end
 
@@ -469,8 +648,83 @@ module Iriq
         load_cluster(key)
       end
 
+      # What Corpus#classify reads: counts, not every value tracked at the
+      # position.
+      def position_evidence(position, value)
+        where = [position.host || "", position.scope.to_s, position.locator]
+        total = @db.get_first_value(
+          "SELECT total FROM position_stats WHERE host = ? AND scope = ? AND locator = ?", where,
+        )
+        return nil if total.nil?
+
+        type_counts = Hash.new(0)
+        @db.execute(
+          "SELECT type, count FROM position_types WHERE host = ? AND scope = ? AND locator = ?", where,
+        ) { |r| type_counts[r[0].to_sym] = coerce_int!(r[1], 1, "count") }
+        value_count = @db.get_first_value(
+          "SELECT count FROM position_values WHERE host = ? AND scope = ? AND locator = ? AND value = ?",
+          [*where, value],
+        )
+        PositionEvidence.new(
+          total:       coerce_int!(total, 0, "total"),
+          type_counts: type_counts,
+          cardinality: @db.get_first_value(
+            "SELECT COUNT(*) FROM position_values WHERE host = ? AND scope = ? AND locator = ?", where,
+          ),
+          value_count: value_count.nil? ? nil : coerce_int!(value_count, 0, "count"),
+        )
+      end
+
+      # One query param's stats — narrower than cluster_for, which also loads
+      # the cluster's examples and segment counts.
+      def param_stats(key, name)
+        total = @db.get_first_value(
+          "SELECT total FROM cluster_params WHERE cluster_key = ? AND name = ?", [key, name],
+        )
+        return nil if total.nil?
+
+        stats = PositionStats.new(max_values: @max_values_per_position)
+        stats.instance_variable_set(:@total, coerce_int!(total, 0, "total"))
+        @db.execute(
+          "SELECT value, count FROM cluster_param_values WHERE cluster_key = ? AND name = ?", [key, name],
+        ) { |r| stats.value_counts[r[0]] = coerce_int!(r[1], 1, "count") }
+        @db.execute(
+          "SELECT type, count FROM cluster_param_types WHERE cluster_key = ? AND name = ?", [key, name],
+        ) { |r| stats.type_counts[r[0].to_sym] = coerce_int!(r[1], 1, "count") }
+        recompute_numeric!(stats)
+        stats
+      end
+
       private
 
+      # Whether remembered value counts can be trusted: under the write lock,
+      # or while the views written are a rebuild nothing else writes.
+      def counts_exact?
+        @in_transaction || @rebuilding
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      # Converting a rollback-mode database to WAL takes an exclusive lock,
+      # and SQLite does NOT consult the busy handler for that lock — so
+      # concurrent first-opens of a fresh corpus can fail with SQLITE_BUSY
+      # even with busy_timeout set. WAL is a persistent database property:
+      # retry briefly — either this connection wins the conversion or
+      # another process already converted the file.
+      def enable_wal!
+        deadline = monotonic_now + LOCK_WAIT
+        begin
+          @db.execute("PRAGMA journal_mode = WAL")
+        rescue SQLite3::BusyException
+          raise if monotonic_now > deadline
+
+          sleep 0.01
+          retry
+        end
+      end
+
       def upsert_shape(table, shape)
         @db.execute(<<~SQL, shape)
           INSERT INTO #{table} (shape, count) VALUES (?, 1)
@@ -480,21 +734,45 @@ module Iriq
 
       def rows_to_count_hash(table, key_col)
         h = Hash.new(0)
-        @db.execute("SELECT #{key_col}, count FROM #{table}") { |r| h[r[0]] = r[1] }
+        @db.execute("SELECT #{key_col}, count FROM #{table}") { |r| h[r[0]] = coerce_int!(r[1], 1, "count") }
         h
       end
 
+      # rusqlite's storage class name for a value SQLite handed back untyped
+      # for an INTEGER column — matches the words rust/ raises for the same
+      # corrupted row so the two runtimes agree.
+      SQLITE_TYPE_NAMES = {
+        NilClass => "Null",
+        Float    => "Real",
+        String   => "Text",
+      }.freeze
+
+      # Every count/total column is declared INTEGER, but SQLite's affinity
+      # rules only convert a value on write when that can be done losslessly
+      # — a hand-edited (or otherwise corrupted) corpus can still leave a
+      # Text or Real value behind. Reads validate the type here instead of
+      # crashing downstream on Integer#+ / Array#sum / Integer#times.
+      def coerce_int!(value, index, name)
+        return value if value.is_a?(Integer)
+
+        type_name = SQLITE_TYPE_NAMES.fetch(value.class, "Blob")
+        raise CorpusError, "corpus #{@path}: Invalid column type #{type_name} at index: #{index}, name: #{name}"
+      end
+
       def load_cluster(key)
+        # key isn't re-selected — it's already the query parameter (matches
+        # the Rust reader's column list, so a corrupted `count` reports the
+        # same column index in both runtimes).
         row = @db.get_first_row(
-          "SELECT key, host, scheme, shape, count FROM clusters WHERE key = ?", [key],
+          "SELECT host, scheme, shape, count FROM clusters WHERE key = ?", [key],
         )
         return nil unless row
 
         c = Cluster.new(
-          key: row[0], host: row[1], scheme: row[2], shape: row[3],
+          key: key, host: row[0], scheme: row[1], shape: row[2],
           max_values: @max_values_per_position,
         )
-        c.instance_variable_set(:@count, row[4])
+        c.instance_variable_set(:@count, coerce_int!(row[3], 3, "count"))
 
         examples = []
         @db.execute(
@@ -507,9 +785,9 @@ module Iriq
           "SELECT position, value, count FROM cluster_segments WHERE cluster_key = ? ORDER BY position",
           [key],
         ) do |r|
-          pos = r[0]
+          pos = coerce_int!(r[0], 0, "position")
           seg_counts[pos] ||= Hash.new(0)
-          seg_counts[pos][r[1]] = r[2]
+          seg_counts[pos][r[1]] = coerce_int!(r[2], 2, "count")
         end
         c.instance_variable_set(:@segment_counts, seg_counts)
 
@@ -522,25 +800,54 @@ module Iriq
           # and type counts; only @total needs filling here. The followup
           # SELECTs below populate value/type rows in place.
           stats = PositionStats.new(max_values: @max_values_per_position)
-          stats.instance_variable_set(:@total, r[1])
+          stats.instance_variable_set(:@total, coerce_int!(r[1], 1, "total"))
           params[r[0]] = stats
         end
         @db.execute(
           "SELECT name, value, count FROM cluster_param_values WHERE cluster_key = ?", [key],
         ) do |r|
           stats = params[r[0]] or next
-          stats.value_counts[r[1]] = r[2]
+          stats.value_counts[r[1]] = coerce_int!(r[2], 2, "count")
         end
         @db.execute(
           "SELECT name, type, count FROM cluster_param_types WHERE cluster_key = ?", [key],
         ) do |r|
           stats = params[r[0]] or next
-          stats.type_counts[r[1].to_sym] = r[2]
+          stats.type_counts[r[1].to_sym] = coerce_int!(r[2], 2, "count")
         end
+        params.each_value { |stats| recompute_numeric!(stats) }
         c.instance_variable_set(:@param_stats, params)
 
         c
       end
+
+      # The rolling numeric aggregates (count/min/max/sum) aren't stored in
+      # the schema — rebuild them from the tracked value counts, mirroring
+      # the Rust backend: only positions with integer/float observations
+      # qualify, and cap-trimmed values are lost.
+      def recompute_numeric!(stats)
+        return if (stats.type_counts[:integer] + stats.type_counts[:float]).zero?
+
+        count = 0
+        min   = nil
+        max   = nil
+        sum   = 0.0
+        stats.value_counts.each do |value, n|
+          num = Float(value, exception: false)
+          next unless num&.finite? # same rule as PositionStats#record_numeric
+
+          count += n
+          min = num if min.nil? || num < min
+          max = num if max.nil? || num > max
+          n.times { sum += num }
+        end
+        return if count.zero?
+
+        stats.instance_variable_set(:@numeric_count, count)
+        stats.instance_variable_set(:@numeric_min, min)
+        stats.instance_variable_set(:@numeric_max, max)
+        stats.instance_variable_set(:@numeric_sum, sum)
+      end
     end
   end
 end

data/lib/iriq/storage.rb

@@ -13,6 +13,10 @@ module Iriq
   module Storage
     SQLITE_EXTS = %w[.db .sqlite .sqlite3].freeze
 
+    TEMP_SEQ_LOCK = Mutex.new
+    @temp_seq = 0
+    def self.next_temp_seq = TEMP_SEQ_LOCK.synchronize { @temp_seq += 1 }
+
     module_function
 
     # Opens (or creates) a storage at `path`, picking the backend by extension.
@@ -29,6 +33,20 @@ module Iriq
         Json.open(path, classifier: classifier, max_values_per_position: max_values_per_position)
       end
     end
+
+    # Replace `path` atomically via a per-write temp file, PATH.<pid>.<n>.tmp
+    # (Rust's writer uses the same shape; --reset sweeps exactly it), + rename.
+    # A shared PATH.tmp let concurrent writers rename each other's file away
+    # (ENOENT). Last writer still wins: a JSON corpus is single-writer.
+    def write_atomically(path, contents)
+      tmp = "#{path}.#{Process.pid}.#{Storage.next_temp_seq}.tmp"
+      File.write(tmp, contents)
+      File.rename(tmp, path)
+    rescue SystemCallError => e
+      raise CorpusError, "corpus #{path}: #{Iriq.os_error_message(e)}"
+    ensure
+      File.delete(tmp) if tmp && File.exist?(tmp)
+    end
   end
 end