nyxis 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9bd1872ba3ef0efd235fc272ba37c6aee39e556f53ca2945fbaee06bd841b8a4
-  data.tar.gz: 251aa729cf14696afc55b02e932d4b5d2d1f4c96b9e0e2a7cc209c8f92fe7c45
+  metadata.gz: 21f182240eeb38df31bb91d81c52a41df156a234b82e61454b49c34f49de5ac8
+  data.tar.gz: ce6fca2da3243e57488f081ee10302cf780575591b2947d11a7becc85ad4feea
 SHA512:
-  metadata.gz: c184c7035fa80b665290cf86841216a16a007a9fbd65129fad9984562fee38ebc67ce5e0a0ab0123c16d02d13d0e2d1221c9c7bed23d66c661bc52638b4e1ede
-  data.tar.gz: cf362b97c76c51a6bed1fc744e1da723c1a728a3c10e3ac7067cd67bbc45e7589d292cadbc30e763b08129f13baa5c9fb5d2c99536dcf79d952728949cdd233f
+  metadata.gz: 27d92d5aad9aa3f661c610666361cd420e337b1955891abd006d02ec899f4b94e7040c5baeb4614b2d71efe457bb6728806a2c7e7f0e6ccda09075416a0286d7
+  data.tar.gz: d5f7df6269b0fe00e8ae6d6512cc67eed7ba33891654b51833f8519a627ac29136f90cd6e0bcceae3cf01db0ef025166bbc84e836a00cf6bf489eafb744ea4d5

data/nxs.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'pattern'
+
 # NXS Reader — .nxb parser (Ruby 3.x, stdlib only).
 #
 # Implements Nyxis v1.1 binary wire format.
@@ -35,6 +37,221 @@ module Nxs
   COL_TAIL_ENTRY_BYTES = 20
   PAX_TAIL_ENTRY_BYTES = 28
 
+  # Adaptive prefetch (phase 1) — spec §6–§8.4
+  DEFAULT_PAGE_SIZE = 65_536
+  DEFAULT_MAX_PAGES = 64
+  DEFAULT_COALESCE_GAP_PAGES = 1
+  DEFAULT_PREFETCH_DEPTH = 4
+  EAGER_THRESHOLD_MB = 10
+  LAZY_THRESHOLD_MB = 50
+
+  HINT_UNKNOWN = 0
+  HINT_SEQUENTIAL = 1
+  HINT_RANDOM = 2
+  HINT_FULL = 3
+  HINT_PARTIAL = 4
+
+  HINT_SYMBOLS = {
+    unknown: HINT_UNKNOWN,
+    sequential: HINT_SEQUENTIAL,
+    random: HINT_RANDOM,
+    full: HINT_FULL,
+    partial: HINT_PARTIAL
+  }.freeze
+
+  def self.normalize_hint(hint)
+    return hint if hint.is_a?(Integer)
+
+    HINT_SYMBOLS.fetch(hint) { HINT_UNKNOWN }
+  end
+
+  # Initial prefetch strategy from open hint and file size (spec §5.1).
+  def self.initial_strategy(hint, file_size)
+    hint = normalize_hint(hint)
+    file_size_mb = file_size / (1024 * 1024)
+    return 'eager' if hint == HINT_FULL && file_size_mb <= EAGER_THRESHOLD_MB
+    return 'lazy' if file_size_mb > LAZY_THRESHOLD_MB
+
+    'adaptive'
+  end
+
+  # Row-layout data sector byte range [start, length).
+  def self.row_data_sector(tail_start, file_size)
+    sector_start = 32
+    if tail_start > sector_start && tail_start <= file_size
+      [sector_start, tail_start - sector_start]
+    else
+      [sector_start, 0]
+    end
+  end
+
+  # Merge sorted unique page indices when gap <= gap_pages (inclusive).
+  def self.coalesce_page_indices(indices, gap_pages, page_size = DEFAULT_PAGE_SIZE)
+    return [] if indices.empty?
+
+    uniq = indices.uniq.sort
+    spans = []
+    start = uniq[0]
+    end_ = uniq[0]
+    uniq.each_cons(2) do |_a, b|
+      if b - end_ <= gap_pages
+        end_ = b
+      else
+        spans << [start, end_]
+        start = end_ = b
+      end
+    end
+    spans << [start, end_]
+    spans.map do |a, b|
+      { page_start: a, page_end: b, byte_start: a * page_size, byte_length: (b - a + 1) * page_size }
+    end
+  end
+
+  def self.clamp_page_ranges(ranges, file_size)
+    ranges.filter_map do |r|
+      len = r[:byte_length]
+      len = file_size - r[:byte_start] if r[:byte_start] + len > file_size
+      next nil if len <= 0
+
+      r.merge(byte_length: len)
+    end
+  end
+
+  def self.page_indices_for_viewport(start_index, end_index, page_size, &record_offset)
+    (start_index..end_index).map { |i| record_offset.call(i) / page_size }
+  end
+
+  # LRU page cache with optional pinning (spec §6).
+  class PageCache
+    attr_reader :max_pages, :page_size, :hits, :misses
+
+    def initialize(max_pages = DEFAULT_MAX_PAGES, page_size = DEFAULT_PAGE_SIZE)
+      @max_pages = max_pages
+      @page_size = page_size
+      @pages = {}
+      @clock = 0
+      @hits = 0
+      @misses = 0
+    end
+
+    def has?(page_index)
+      @pages.key?(page_index)
+    end
+
+    def get(page_index)
+      entry = @pages[page_index]
+      unless entry
+        @misses += 1
+        return nil
+      end
+      @clock += 1
+      entry[:last_used] = @clock
+      @hits += 1
+      entry[:data]
+    end
+
+    def set(page_index, data, pinned: false)
+      return if @max_pages <= 0
+
+      while @pages.size >= @max_pages && !evict_one?; end
+      @clock += 1
+      @pages[page_index] = { data: data, last_used: @clock, pinned: pinned }
+    end
+
+    def pin_pages(page_indices)
+      page_indices.each do |p|
+        entry = @pages[p]
+        entry[:pinned] = true if entry
+      end
+    end
+
+    def unpin_all
+      @pages.each_value { |entry| entry[:pinned] = false }
+    end
+
+    def stats
+      bytes = @pages.values.sum { |e| e[:data].bytesize }
+      {
+        pages_cached: @pages.size,
+        pages_max: @max_pages,
+        memory_used_bytes: bytes,
+        cache_hits: @hits,
+        cache_misses: @misses
+      }
+    end
+
+    private
+
+    def evict_one?
+      victim = nil
+      oldest = nil
+      @pages.each do |idx, entry|
+        next if entry[:pinned]
+
+        if oldest.nil? || entry[:last_used] < oldest
+          oldest = entry[:last_used]
+          victim = idx
+        end
+      end
+      return false unless victim
+
+      @pages.delete(victim)
+      true
+    end
+  end
+
+  # In-flight page fetch deduplication for concurrent prefetch_viewport calls.
+  class InFlightMap
+    Entry = Struct.new(:queue, :data, :error)
+
+    def initialize
+      @mu = Mutex.new
+      @map = {}
+    end
+
+    def has?(page_index)
+      @mu.synchronize { @map.key?(page_index) }
+    end
+
+    def wait(page_index)
+      entry = @mu.synchronize { @map[page_index] }
+      return nil unless entry
+
+      entry.queue.pop
+      raise entry.error if entry.error
+
+      entry.data
+    end
+
+    def with(page_index)
+      entry = nil
+      leader = @mu.synchronize do
+        existing = @map[page_index]
+        if existing
+          false
+        else
+          entry = Entry.new(Queue.new)
+          @map[page_index] = entry
+          true
+        end
+      end
+      return wait(page_index) unless leader
+
+      begin
+        data = yield
+        entry.data = data
+        entry.queue << true
+        data
+      rescue StandardError => e
+        entry.error = e
+        entry.queue << true
+        raise
+      ensure
+        @mu.synchronize { @map.delete(page_index) if @map[page_index] == entry }
+      end
+    end
+  end
+
   class NxsError < StandardError
     attr_reader :code
 
@@ -49,7 +266,13 @@ module Nxs
   class Reader
     attr_reader :keys, :record_count, :layout
 
-    def initialize(bytes)
+    def initialize(bytes, **options)
+      hint = options.fetch(:hint, HINT_UNKNOWN)
+      max_pages = options.fetch(:max_pages, DEFAULT_MAX_PAGES)
+      page_size = options.fetch(:page_size, DEFAULT_PAGE_SIZE)
+      coalesce_gap_pages = options.fetch(:coalesce_gap_pages, DEFAULT_COALESCE_GAP_PAGES)
+      prefetch_depth = options.fetch(:prefetch_depth, DEFAULT_PREFETCH_DEPTH)
+      fetch_range = options.fetch(:fetch_range, nil)
       @data = bytes.b # force binary encoding
       sz = @data.bytesize
       raise NxsError.new('ERR_OUT_OF_BOUNDS', 'file too small') if sz < 32
@@ -86,6 +309,15 @@ module Nxs
       @col_buf_off = []
       @col_buf_len = []
       parse_layout_tail!(preamble_tail)
+      init_column_prefetch!(fetch_range: fetch_range)
+      init_prefetch!(
+        hint: hint,
+        max_pages: max_pages,
+        page_size: page_size,
+        coalesce_gap_pages: coalesce_gap_pages,
+        prefetch_depth: prefetch_depth,
+        fetch_range: fetch_range
+      )
     end
 
     # O(1) record lookup — row tail-index or columnar/PAX record index.
@@ -96,10 +328,41 @@ module Nxs
 
       return Object.new(self, i, i) if @layout != :row
 
+      on_access(i)
       abs_offset = @data.unpack1("@#{@tail_start + i * 10 + 2}Q<")
       Object.new(self, abs_offset)
     end
 
+    # Prefetch one column buffer (columnar layout only; §7.4).
+    def prefetch_column(key)
+      raise NxsError.new('ERR_LAYOUT', 'prefetch_column requires columnar layout') unless @layout == :columnar
+
+      slot = @key_index[key]
+      raise NxsError.new('ERR_KEY_NOT_FOUND', "key #{key.inspect} not in schema") unless slot
+
+      off = nil
+      length = nil
+      fetch = nil
+      @col_mu.synchronize do
+        return if @col_warmed[slot]
+
+        off = @col_buf_off[slot].to_i
+        length = @col_buf_len[slot].to_i
+        raise NxsError.new('ERR_OUT_OF_BOUNDS', 'column buffer') if off.negative? || length.negative?
+        raise NxsError.new('ERR_OUT_OF_BOUNDS', 'column buffer') if !@col_remote_fetch && off + length > @data.bytesize
+
+        fetch = @col_fetch_range
+      end
+      blob = fetch.call(off, length)
+      @col_mu.synchronize do
+        return if @col_warmed[slot]
+
+        @col_overlay[slot] = blob if off + blob.bytesize > @data.bytesize
+        @col_warmed[slot] = true
+        @col_fetches += 1
+      end
+    end
+
     # Sum f64 column — columnar/PAX buffer path or row scan.
     def sum_f64(key)
       return col_sum_f64(key) if @layout != :row
@@ -282,8 +545,255 @@ module Nxs
       end
     end
 
+    # rubocop:disable Metrics/ParameterLists -- prefetch open options mirror Go OpenOptions
+    def init_prefetch!(hint:, max_pages:, page_size:, coalesce_gap_pages:, prefetch_depth:, fetch_range:)
+      @prefetch_mu = Mutex.new
+      @cache_mu = Mutex.new
+      @prefetch_hint = Nxs.normalize_hint(hint)
+      @prefetch_page_size = page_size
+      @prefetch_depth = prefetch_depth.positive? ? prefetch_depth : DEFAULT_PREFETCH_DEPTH
+      @coalesce_gap_pages = coalesce_gap_pages
+      @page_cache = PageCache.new(max_pages, page_size)
+      @in_flight = InFlightMap.new
+      @fetches_issued = 0
+      @detector = AccessPatternDetector.new
+      @prefetch_strategy = Nxs.initial_strategy(@prefetch_hint, @data.bytesize)
+      @prefetch_pattern = PATTERN_UNKNOWN
+      @eager_started = false
+      @eager_complete = false
+      @eager_cancel = false
+      @eager_thread = nil
+      @closed = false
+      @prefetch_paused = false
+      @fetch_range = fetch_range || lambda do |byte_start, byte_length|
+        raise NxsError.new('ERR_OUT_OF_BOUNDS', 'fetch range out of bounds') if byte_start.negative?
+
+        end_ = byte_start + byte_length
+        raise NxsError.new('ERR_OUT_OF_BOUNDS', 'fetch range out of bounds') if end_ > @data.bytesize
+
+        @data[byte_start, byte_length]
+      end
+      start_eager_background! if @layout == :row && @prefetch_strategy == 'eager'
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    # Block until eager / background prefetch completes (spec §8).
+    def warmup
+      t = @prefetch_mu.synchronize { @eager_thread }
+      t&.join
+    end
+
+    # Stop scheduling speculative and eager prefetch (§8.1).
+    def pause_prefetch
+      @prefetch_mu.synchronize { @prefetch_paused = true }
+    end
+
+    # Re-enable speculative prefetch after pause_prefetch.
+    def resume_prefetch
+      @prefetch_mu.synchronize { @prefetch_paused = false }
+    end
+
+    # Cancel in-flight eager prefetch and wait for the background thread.
+    def close
+      t = nil
+      @prefetch_mu.synchronize do
+        @closed = true
+        @eager_cancel = true
+        t = @eager_thread
+      end
+      t&.join
+    end
+
+    def on_access(index)
+      return unless @layout == :row
+      return if @record_count.zero?
+
+      adaptive_seq = false
+      skip_spec = false
+      start_eager = false
+      @prefetch_mu.synchronize do
+        return if @closed || @prefetch_paused
+
+        @detector.observe(index)
+        @prefetch_pattern = @detector.pattern
+        start_eager = maybe_upgrade_to_eager!
+        if eager_complete? || @prefetch_strategy == 'eager'
+          skip_spec = true
+          next
+        end
+        page_index = record_byte_offset(index) / @prefetch_page_size
+        @cache_mu.synchronize { @page_cache.get(page_index) }
+        adaptive_seq = @prefetch_strategy == 'adaptive' && @detector.pattern == PATTERN_SEQUENTIAL
+      end
+      start_eager_background! if start_eager
+      return if skip_spec
+
+      speculative_prefetch! if adaptive_seq
+    end
+
+    def record_byte_offset(i)
+      @data.unpack1("@#{@tail_start + i * 10 + 2}Q<")
+    end
+
+    # Prefetch pages for records [start_index, end_index] (row layout only).
+    def prefetch_viewport(start_index, end_index)
+      return self if @layout != :row
+
+      n = @record_count
+      unless start_index.between?(0, end_index) && end_index < n
+        raise NxsError.new(
+          'ERR_OUT_OF_BOUNDS',
+          "prefetch_viewport [#{start_index}, #{end_index}] out of [0, #{n})"
+        )
+      end
+
+      @cache_mu.synchronize do
+        page_size = @prefetch_page_size
+        indices = Nxs.page_indices_for_viewport(start_index, end_index, page_size) do |i|
+          record_byte_offset(i)
+        end
+        missing = indices.uniq.select { |p| !@page_cache.has?(p) && !@in_flight.has?(p) }
+        if missing.empty?
+          @page_cache.pin_pages(indices)
+          @page_cache.unpin_all
+          return self
+        end
+
+        ranges = Nxs.clamp_page_ranges(
+          Nxs.coalesce_page_indices(missing, @coalesce_gap_pages, page_size),
+          @data.bytesize
+        )
+        ranges.each { |r| fetch_coalesced_range_unlocked!(r) }
+        @page_cache.pin_pages(indices)
+        @page_cache.unpin_all
+      end
+      self
+    end
+
+    def cache_stats
+      stats = @page_cache.stats
+      col_fetches = @col_mu.synchronize { @col_fetches }
+      strategy, pattern = @prefetch_mu.synchronize do
+        [@prefetch_strategy, @detector.pattern]
+      end
+      stats.merge(
+        fetches_issued: @fetches_issued,
+        column_fetches_issued: col_fetches,
+        strategy: strategy,
+        pattern: pattern
+      )
+    end
+
     private
 
+    def eager_complete?
+      @prefetch_strategy == 'eager' && @eager_complete
+    end
+
+    def maybe_upgrade_to_eager!
+      return if @prefetch_paused
+      return unless @prefetch_strategy == 'adaptive'
+      return unless @detector.pattern == PATTERN_SEQUENTIAL
+      return if @detector.sequential_runs < UPGRADE_SEQUENTIAL_THRESHOLD
+      return if @data.bytesize / (1024 * 1024) > EAGER_THRESHOLD_MB
+
+      @prefetch_strategy = 'eager'
+      true
+    end
+
+    def speculative_prefetch!
+      return if @prefetch_mu.synchronize { @prefetch_paused }
+
+      predicted = @prefetch_mu.synchronize { @detector.predict_next(@prefetch_depth, @record_count) }
+      return if predicted.empty?
+
+      page_size = @prefetch_page_size
+      missing = @cache_mu.synchronize do
+        predicted.filter_map do |idx|
+          off = record_byte_offset(idx)
+          p = off / page_size
+          p unless @page_cache.has?(p) || @in_flight.has?(p)
+        end.uniq
+      end
+      return if missing.empty?
+
+      ranges = Nxs.clamp_page_ranges(
+        Nxs.coalesce_page_indices(missing, @coalesce_gap_pages, page_size),
+        @data.bytesize
+      )
+      ranges.each { |r| fetch_coalesced_range!(r) }
+    end
+
+    def start_eager_background!
+      return unless @prefetch_strategy == 'eager'
+
+      @prefetch_mu.synchronize do
+        return if @prefetch_paused || @eager_started
+
+        @eager_started = true
+        sector_start, sector_len = Nxs.row_data_sector(@tail_start, @data.bytesize)
+        if sector_len.zero?
+          @eager_complete = true
+          next
+        end
+        @eager_thread = Thread.new { run_eager_background(sector_start, sector_len) }
+      end
+    end
+
+    def run_eager_background(sector_start, sector_len)
+      end_byte = [sector_start + sector_len, @data.bytesize].min
+      return if sector_start >= end_byte
+
+      page_size = @prefetch_page_size
+      first_page = sector_start / page_size
+      last_page = (end_byte - 1) / page_size
+      indices = (first_page..last_page).to_a
+      eager_cancelled = @prefetch_mu.synchronize { @eager_cancel }
+      return if eager_cancelled
+
+      missing = @cache_mu.synchronize do
+        indices.select { |p| !@page_cache.has?(p) && !@in_flight.has?(p) }
+      end
+      if missing.empty?
+        @prefetch_mu.synchronize { @eager_complete = true unless @eager_cancel }
+        return
+      end
+
+      ranges = Nxs.clamp_page_ranges(
+        Nxs.coalesce_page_indices(missing, @coalesce_gap_pages, page_size),
+        @data.bytesize
+      )
+      ranges.each do |r|
+        break if @prefetch_mu.synchronize { @eager_cancel }
+
+        fetch_coalesced_range!(r)
+      end
+      @prefetch_mu.synchronize { @eager_complete = true unless @eager_cancel }
+    end
+
+    def fetch_coalesced_range!(page_range)
+      @cache_mu.synchronize { fetch_coalesced_range_unlocked!(page_range) }
+    end
+
+    def fetch_coalesced_range_unlocked!(page_range)
+      blob = fetch_range_bytes!(page_range[:byte_start], page_range[:byte_length])
+      page_size = @prefetch_page_size
+      (page_range[:page_start]..page_range[:page_end]).each do |p|
+        next if @page_cache.has?(p)
+
+        page_off = p * page_size - page_range[:byte_start]
+        page_len = [page_size, blob.bytesize - page_off].min
+        next if page_len <= 0
+
+        @page_cache.set(p, blob[page_off, page_len])
+      end
+    end
+
+    def fetch_range_bytes!(byte_start, byte_length)
+      @fetches_issued += 1
+      @fetch_range.call(byte_start, byte_length)
+    end
+
     def parse_layout_tail!(preamble_tail)
       if (@flags & FLAG_COLUMNAR != 0) && (@flags & FLAG_PAX != 0)
         raise NxsError.new('ERR_INVALID_FLAGS', 'columnar and PAX both set')
@@ -422,17 +932,39 @@ module Nxs
     end
 
     def col_field_parts(slot)
+      sector = column_sector(slot)
+      bm_len = null_bitmap_bytes(@record_count)
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'null bitmap') if sector.bytesize < bm_len
+
+      [sector[0, bm_len], sector[bm_len..]]
+    end
+
+    def init_column_prefetch!(fetch_range: nil)
+      return unless @layout == :columnar
+
+      @col_mu = Mutex.new
+      @col_warmed = {}
+      @col_overlay = {}
+      @col_fetches = 0
+      @col_remote_fetch = !fetch_range.nil?
+      data = @data
+      @col_fetch_range = fetch_range || ->(off, len) { data[off, len] }
+    end
+
+    def column_sector(slot)
       raise NxsError.new('ERR_OUT_OF_BOUNDS', "key slot #{slot}") if slot.negative? || slot >= @col_buf_off.length
 
       off = @col_buf_off[slot].to_i
       length = @col_buf_len[slot].to_i
+      if @col_warmed
+        @col_mu.synchronize do
+          overlay = @col_overlay[slot]
+          return overlay[0, length] if @col_warmed[slot] && overlay && !overlay.empty?
+        end
+      end
       raise NxsError.new('ERR_OUT_OF_BOUNDS', 'column buffer') if off + length > @data.bytesize
 
-      bm_len = null_bitmap_bytes(@record_count)
-      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'null bitmap') if length < bm_len
-
-      sector = @data[off, length]
-      [sector[0, bm_len], sector[bm_len..]]
+      @data[off, length]
     end
 
     def col_var_parts(slot)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nyxis
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Micael Malta
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-05-22 00:00:00.000000000 Z
+date: 2026-05-24 00:00:00.000000000 Z
 dependencies: []
 description: |
   Pure-Ruby reader for NXB files produced by the NXS compiler. Provides