insika 0.8.0 → 1.0.1

data/lib/insika/stores/sqlite.rb

@@ -1,8 +1,11 @@
 # frozen_string_literal: true
 
 require "json"
+require "logger"
 require "time"
 require "async/semaphore"
+require "zlib"
+require "async/promise"
 
 module Insika
   module Stores
@@ -11,34 +14,89 @@ module Insika
     # writes in a transaction serialized by an Async::Semaphore.
     class SQLite
       include Store
+      include JsonValue
 
+      # Prepended per instance when slow_ms is set; see #initialize.
+      module SlowLog
+        %i[get set delete list entries recent scopes].each do |name|
+          define_method(name) do |*args, &blk|
+            t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            result = super(*args, &blk)
+            ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000
+            report_slow(name, args.first, ms) if ms >= @slow_ms
+            result
+          end
+        end
+
+        private
+
+
+        def report_slow(name, scope, ms)
+          # up to 3 frames of the caller chain (the store method alone rarely says
+          # which duty called it), innermost first
+          chain = caller_locations(2, 40).reject { |l| l.path.end_with?("stores/sqlite.rb") }
+                                        .select { |l| l.path.include?("/insika/") }.first(3)
+          @slow_logger.warn(format("slow store %s scope=%s %.1f ms at %s", name, scope, ms,
+                                   chain.map { |l| "#{File.basename(l.path)}:#{l.lineno}" }.join(" < ")))
+        end
+      end
+
+      # A rowid table plus a (scope, key) index, not WITHOUT ROWID: values run to
+      # tens of KB, and a WITHOUT ROWID table keeps each row inside the key
+      # b-tree, so every key scan (list, the prefix ranges) read the values too.
+      # The index holds only the keys; a value is read when it is asked for.
+      # SQLite's own guidance: WITHOUT ROWID is for small rows.
       DDL = <<~SQL
         CREATE TABLE IF NOT EXISTS kv (
           scope      TEXT    NOT NULL,
           key        TEXT    NOT NULL,
           value      TEXT    NOT NULL,
-          updated_at TEXT    NOT NULL,
-          PRIMARY KEY (scope, key)
-        ) WITHOUT ROWID;
+          updated_at TEXT    NOT NULL
+        );
+        CREATE UNIQUE INDEX IF NOT EXISTS kv_scope_key ON kv (scope, key);
       SQL
 
-      # JSON model types + Symbol (coerced to String on write).
-      # Any other type is "garbage" and must be rejected. Identical to
-      # Stores::Memory — both backends share the SAME type model
-      # (the contract suite is honest).
-      JSONABLE = [NilClass, TrueClass, FalseClass, Integer, Float,
-                  String, Symbol].freeze
-      private_constant :JSONABLE
+      # The rebuild of a file still in the old layout: one transaction, so a crash
+      # leaves the old table whole. Rows go in by updated_at so their rowids follow
+      # write order, which #recent reads them in.
+      REBUILD = <<~SQL
+        CREATE TABLE kv_rowid (
+          scope      TEXT    NOT NULL,
+          key        TEXT    NOT NULL,
+          value      TEXT    NOT NULL,
+          updated_at TEXT    NOT NULL
+        );
+        INSERT INTO kv_rowid (scope, key, value, updated_at)
+          SELECT scope, key, value, updated_at FROM kv ORDER BY scope, updated_at, key;
+        DROP TABLE kv;
+        ALTER TABLE kv_rowid RENAME TO kv;
+      SQL
 
       # lazy require: the core installs without the sqlite3 gem
       # when only Memory is used.
-      def initialize(path:, serializer: JSON)
+      # autocheckpoint: false when Litestream replicates this file — it owns the
+      # checkpoints, and the app's own (every 1000 pages, per connection, N
+      # workers) race it: the WAL grows past the database itself while Litestream's
+      # checkpoint fails "database is locked". litestream.io/tips.
+      # slow_ms: log every call that takes at least this long (nil = off, zero
+      # cost). A call holds the GVL for its whole duration, so under a fiber
+      # server a slow one stalls every turn in the process; the line names the
+      # method, scope and Insika caller so the offender is found, not guessed.
+      def initialize(path:, serializer: JSON, autocheckpoint: true, slow_ms: nil, logger: nil)
         require "sqlite3"
 
+        if slow_ms
+          @slow_ms = slow_ms
+          @slow_logger = logger || Logger.new($stderr)
+          singleton_class.prepend(SlowLog)
+        end
+
         @serializer = serializer
         @db = SQLite3::Database.new(path)
         @write_semaphore = Async::Semaphore.new(1)
         @tx_owner = nil
+        @pending_writes = []
+        @flushing = false
 
         # Multi-process boot (N Falcon workers opening the SAME file at the
         # same time): `PRAGMA journal_mode = WAL` on a new file needs an
@@ -58,10 +116,10 @@ module Insika
         with_busy_retry do
           @db.execute("PRAGMA journal_mode = WAL")
           @db.execute("PRAGMA synchronous = NORMAL")
+          @db.execute("PRAGMA wal_autocheckpoint = 0") unless autocheckpoint
+          rebuild_without_rowid!
           @db.execute_batch(DDL)
         end
-        # The WITHOUT ROWID PRIMARY KEY (scope, key) is already the prefix index —
-        # there is no extra index to create.
       rescue ::SQLite3::Exception => e
         raise Insika::StoreError, "failed to open #{path}: #{e.message}"
       end
@@ -86,6 +144,26 @@ module Insika
       end
       private :with_busy_retry
 
+      # Rebuilds a WITHOUT ROWID kv (the old layout) into the rowid one. Checked
+      # again under BEGIN IMMEDIATE: of N workers opening the file together only
+      # the first rebuilds; the rest find it done. On a big file this takes
+      # seconds, so a deployment should open the store once before its workers
+      # (deploy/entrypoint.sh does).
+      def rebuild_without_rowid!
+        return unless without_rowid?
+
+        @db.transaction(:immediate) do
+          @db.execute_batch(REBUILD) if without_rowid?
+        end
+      end
+      private :rebuild_without_rowid!
+
+      def without_rowid?
+        @db.get_first_value("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'kv'")
+           .to_s.include?("WITHOUT ROWID")
+      end
+      private :without_rowid?
+
       def close
         @db&.close
         nil
@@ -95,14 +173,14 @@ module Insika
         row = @db.get_first_value(
           "SELECT value FROM kv WHERE scope = ? AND key = ?", [scope, key]
         )
-        row.nil? ? nil : @serializer.parse(row)
+        row.nil? ? nil : @serializer.parse(decode(row))
       rescue ::SQLite3::Exception => e
         raise Insika::StoreError, e.message
       end
 
       def set(scope, key, value)
         serialized = serialize(value)               # fail-fast BEFORE writing
-        transaction do
+        group_write(scope, serialized.bytesize) do
           @db.execute(
             "INSERT OR REPLACE INTO kv (scope, key, value, updated_at) " \
             "VALUES (?, ?, ?, ?)",
@@ -112,22 +190,96 @@ module Insika
         value
       end
 
+      # An UPDATE keeps the row's rowid, the write order #recent reads.
+      def amend(scope, key, value)
+        serialized = serialize(value)
+        updated = group_write(scope, serialized.bytesize) do
+          @db.execute("UPDATE kv SET value = ? WHERE scope = ? AND key = ?", [serialized, scope, key])
+          @db.changes.positive?
+        end
+        updated ? value : set(scope, key, value)
+      end
+
       def delete(scope, key)
-        transaction do
+        group_write(scope, 0) do
           @db.execute("DELETE FROM kv WHERE scope = ? AND key = ?", [scope, key])
           @db.changes.positive?
         end
       end
 
+      # A prefix is a primary-key RANGE [prefix, next): SQLite seeks straight to
+      # it. Filtering the whole scope in Ruby read every row of it (values of tens
+      # of KB in a WITHOUT ROWID table) while holding the GVL, so its cost grew with
+      # the database and, called on every turn, starved the reactor.
+      LIST_PREFIX_SQL = "SELECT key FROM kv WHERE scope = ? AND key >= ? AND key < ? ORDER BY key"
+
       def list(scope, prefix = nil)
-        keys = @db.execute(
-          "SELECT key FROM kv WHERE scope = ? ORDER BY key", [scope]
-        ).map(&:first)
+        upper = prefix && range_end(prefix)
+        return @db.execute(LIST_PREFIX_SQL, [scope, prefix, upper]).map(&:first) if upper
+
+        keys = @db.execute("SELECT key FROM kv WHERE scope = ? ORDER BY key", [scope]).map(&:first)
         prefix ? keys.select { |k| k.start_with?(prefix) } : keys
       rescue ::SQLite3::Exception => e
         raise Insika::StoreError, e.message
       end
 
+      # One query for the pairs: a caller listing and reading a whole scope pays
+      # one statement instead of one per key.
+      def entries(scope, prefix = nil)
+        upper = prefix && range_end(prefix)
+        rows =
+          if upper
+            @db.execute("SELECT key, value FROM kv WHERE scope = ? AND key >= ? AND key < ? ORDER BY key", [scope, prefix, upper])
+          else
+            @db.execute("SELECT key, value FROM kv WHERE scope = ? ORDER BY key", [scope])
+          end
+        rows = rows.select { |key, _| key.start_with?(prefix) } if prefix && !upper
+        rows.map { |key, raw| [key, @serializer.parse(decode(raw))] }
+      rescue ::SQLite3::Exception => e
+        raise Insika::StoreError, e.message
+      end
+
+      # Newest first by rowid: every write (INSERT OR REPLACE) takes a new, higher
+      # rowid, so rowid order IS write order — no extra column or index. Two steps:
+      # the page's keys come from the (scope, key) index alone, then only their
+      # values are read. Asking for key and value in one query made SQLite read
+      # every value of the range just to sort it (a temp B-tree over the rows).
+      RECENT_KEYS_PREFIX_SQL = "SELECT key FROM kv WHERE scope = ? AND key >= ? AND key < ? " \
+                               "ORDER BY rowid DESC LIMIT ? OFFSET ?"
+      RECENT_KEYS_SQL = "SELECT key FROM kv WHERE scope = ? ORDER BY rowid DESC LIMIT ? OFFSET ?"
+      VALUES_BATCH = 500 # keys per IN (...) read, well under SQLite's variable limit
+
+      def recent(scope, prefix, limit, offset = 0)
+        upper = prefix && range_end(prefix)
+        if prefix && !upper
+          return @db.execute("SELECT key, value FROM kv WHERE scope = ? ORDER BY rowid DESC", [scope])
+                    .select { |key, _| key.start_with?(prefix) }.drop(offset).first(limit)
+                    .map { |key, raw| [key, @serializer.parse(decode(raw))] }
+        end
+
+        keys = (upper ? @db.execute(RECENT_KEYS_PREFIX_SQL, [scope, prefix, upper, limit, offset])
+                      : @db.execute(RECENT_KEYS_SQL, [scope, limit, offset])).flatten
+        values = {}
+        keys.each_slice(VALUES_BATCH) do |batch|
+          @db.execute("SELECT key, value FROM kv WHERE scope = ? AND key IN (#{Array.new(batch.size, '?').join(',')})",
+                      [scope, *batch]).each { |key, raw| values[key] = raw }
+        end
+        # a key deleted between the two reads simply drops out of the page
+        keys.filter_map { |key| (raw = values[key]) && [key, @serializer.parse(decode(raw))] }
+      rescue ::SQLite3::Exception => e
+        raise Insika::StoreError, e.message
+      end
+
+      # The smallest key above every key starting with `prefix`: its last byte + 1.
+      # ASCII only (every prefix the stores use), so the bound stays valid UTF-8 and
+      # compares byte-wise under BINARY; anything else falls back to the scan.
+      def range_end(prefix)
+        return nil if prefix.empty? || !prefix.ascii_only? || prefix.end_with?("\x7F")
+
+        prefix[0...-1] + (prefix[-1].ord + 1).chr
+      end
+      private :range_end
+
       def scopes(prefix = nil)
         names = @db.execute("SELECT DISTINCT scope FROM kv ORDER BY scope").map(&:first)
         names = names.select { |s| s.start_with?(prefix) } if prefix
@@ -143,7 +295,11 @@ module Insika
 
         @write_semaphore.acquire do
           begin
-            @db.transaction(:immediate)
+            # N workers on one file: BEGIN IMMEDIATE can fail at once, with no
+            # busy-handler wait, when this shared connection has a read open
+            # (another fiber) and a sibling process just committed. Nothing is
+            # written yet, so retrying the BEGIN alone is safe (~5s ceiling).
+            with_busy_retry(attempts: 100, backoff: 0.05) { @db.transaction(:immediate) }
             @tx_owner = Fiber.current
             result = yield
             @db.commit
@@ -161,31 +317,89 @@ module Insika
 
       private
 
-      # Enforces the contract's type model at the boundary:
-      # Symbol/symbol-key become String; a type outside the JSON model ->
-      # StoreError on WRITE (fail-fast; never writes garbage).
-      #
-      # Does NOT use `generate(strict: true)`: under json 2.7.1 (the version that
-      # ships with ruby 3.3.5 pinned in Gemfile.lock) `strict` REJECTS Symbol,
-      # which would violate the Symbol coercion. The explicit validation
-      # is independent of the json version and gives the SAME semantics as Memory.
-      def serialize(value)
-        ensure_jsonable!(value)
-        @serializer.generate(value)
-      rescue JSON::GeneratorError => e
-        raise Insika::StoreError, "value not serializable: #{e.message}"
-      end
-
-      def ensure_jsonable!(value)
-        case value
-        when *JSONABLE then nil
-        when Array then value.each { |v| ensure_jsonable!(v) }
-        when Hash then value.each { |k, v| ensure_jsonable!(k); ensure_jsonable!(v) }
-        else
-          raise Insika::StoreError,
-                "value not serializable: #{value.class} not allowed in JSON"
+      # A BLOB is a value an earlier release stored deflated; JSON is always text.
+      def decode(raw)
+        raw.encoding == Encoding::BINARY ? Zlib::Inflate.inflate(raw).force_encoding(Encoding::UTF_8) : raw
+      end
+
+      # Group commit for the single-statement writes (set/delete). The first one
+      # becomes the leader and takes the write lock; every write that arrives while
+      # it waits (on the semaphore, or on another PROCESS holding the file) joins
+      # the same transaction and one COMMIT. Each caller still returns only after
+      # its write is committed, so durability and per-fiber order are unchanged;
+      # what drops is how often N workers fight over the file's one write lock.
+      # A statement that fails fails only its own caller; one that leaves no
+      # transaction behind (SQLite rolled it back) fails the whole batch. A leader
+      # stopped before it committed hands the queue back: each write re-queues and
+      # one of them leads.
+      LeaderStopped = Class.new(StandardError)
+      private_constant :LeaderStopped
+
+      def group_write(scope, bytes, &op)
+        return op.call if @tx_owner == Fiber.current # inside an explicit transaction
+
+        begin
+          write = [op, Async::Promise.new, scope, bytes]
+          @pending_writes << write
+          flush_writes unless @flushing
+          write[1].wait
+        rescue LeaderStopped
+          retry
+        end
+      end
+
+      def flush_writes
+        @flushing = true
+        until @pending_writes.empty?
+          batch = nil
+          results = nil
+          begin
+            clock = [monotonic]
+            transaction do
+              clock << monotonic # the lock is ours
+              batch = @pending_writes
+              @pending_writes = []
+              results = batch.map { |(op, _)| run_write(op) }
+              raise Insika::StoreError, "the batch's transaction was rolled back" unless @db.transaction_active?
+
+              clock << monotonic
+            end
+            clock << monotonic
+            report_slow_batch(batch, clock) if @slow_ms
+            batch.zip(results) { |(_, promise), (ok, value)| ok ? promise.resolve(value) : promise.reject(value) }
+          rescue StandardError => e
+            (batch || @pending_writes.slice!(0..)).each { |(_, promise)| promise.reject(e) unless promise.completed? }
+          end
+        end
+      ensure
+        @flushing = false
+        # only non-empty when the leader stopped mid-flush (e.g. its task was cancelled)
+        (Array(batch) + @pending_writes.slice!(0..)).each do |(_, promise)|
+          promise.reject(LeaderStopped.new) unless promise.completed?
         end
       end
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      # One line per slow batch: wait = until the write lock was ours (this
+      # process's semaphore and other processes holding the file), exec = the
+      # statements, commit = COMMIT. A long wait with a short commit means
+      # someone else holds the lock; a long commit means the batch itself is heavy.
+      def report_slow_batch(batch, clock)
+        start, locked, executed, committed = clock
+        return if (committed - start) * 1000 < @slow_ms
+
+        scopes = batch.map { |w| w[2].to_s.sub(/:.*/, ":*") }.tally.map { |name, n| "#{name}:#{n}" }.join(",")
+        @slow_logger.warn(format("slow store batch writes=%d bytes=%d wait=%.1f ms exec=%.1f ms commit=%.1f ms scopes=%s",
+                                 batch.size, batch.sum { |w| w[3] }, (locked - start) * 1000,
+                                 (executed - locked) * 1000, (committed - executed) * 1000, scopes))
+      end
+
+      def run_write(op)
+        [true, op.call]
+      rescue ::SQLite3::Exception => e
+        [false, Insika::StoreError.new(e.message)]
+      end
     end
   end
 end

data/lib/insika/stores/turn_counter.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Insika
+  module Stores
+    # Counts every store call into the running turn's TurnTiming, so a turn's
+    # completion event says how many round trips it cost and to which scopes.
+    # Prepended to one backend instance; with no turn running it only forwards.
+    module TurnCounter
+      OPS = %i[get set delete list scopes entries recent transaction].freeze
+
+      OPS.each do |op|
+        define_method(op) do |*args, &blk|
+          Insika::TurnTiming.current&.count_store(op, args.first)
+          super(*args, &blk)
+        end
+      end
+
+      # -> the same backend, counting from now on (idempotent).
+      def self.attach(backend)
+        backend.singleton_class.prepend(self) unless backend.singleton_class.include?(self)
+        backend
+      end
+    end
+  end
+end