nyxis 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ebcbfcd1dfc34288e738e81acc006a8cabda0fdc50e9e851c73e55d422dadb2
-  data.tar.gz: d1eb4b53469a62e382bbf385cf039307af2a5be59b62ba0db0d6f0abe3dffe73
+  metadata.gz: 9bd1872ba3ef0efd235fc272ba37c6aee39e556f53ca2945fbaee06bd841b8a4
+  data.tar.gz: 251aa729cf14696afc55b02e932d4b5d2d1f4c96b9e0e2a7cc209c8f92fe7c45
 SHA512:
-  metadata.gz: 74dcb43ae96b59f6a56b52dd64d03efb2074c8f3789a99fbe1889b26ed931cb9280fd418036d837702a28cf040d79722f7d51a120d43788aa239f1ec6dbb4d35
-  data.tar.gz: 339bf6357eebf0557ea3d114dbe970eb42f9f796f6665db9ca6369a59c2c63237135b2ff4681b5305bea05a15f1fc6b09c8347e8a836244ae53afd9f354966c6
+  metadata.gz: c184c7035fa80b665290cf86841216a16a007a9fbd65129fad9984562fee38ebc67ce5e0a0ab0123c16d02d13d0e2d1221c9c7bed23d66c661bc52638b4e1ede
+  data.tar.gz: cf362b97c76c51a6bed1fc744e1da723c1a728a3c10e3ac7067cd67bbc45e7589d292cadbc30e763b08129f13baa5c9fb5d2c99536dcf79d952728949cdd233f

data/nxs.rb

@@ -22,8 +22,18 @@
 module Nxs
   MAGIC_FILE   = 0x4E595842  # NYXB
   MAGIC_OBJ    = 0x4E59584F  # NYXO
+  MAGIC_LIST   = 0x4E59584C  # NYXL
+  MAGIC_PAGE   = 0x4E585350  # NYXP
   MAGIC_FOOTER = 0x2153584E  # NXS!
-  FLAG_SCHEMA  = 0x0002
+  FLAG_COLUMNAR = 0x0001
+  FLAG_PAX      = 0x0004
+  FLAG_SCHEMA   = 0x0002
+
+  FOOTER_ROW_BYTES = 12
+  FOOTER_COL_BYTES = 20
+  FOOTER_PAX_BYTES = 28
+  COL_TAIL_ENTRY_BYTES = 20
+  PAX_TAIL_ENTRY_BYTES = 28
 
   class NxsError < StandardError
     attr_reader :code
@@ -37,7 +47,7 @@ module Nxs
   # ── Reader ──────────────────────────────────────────────────────────────────
 
   class Reader
-    attr_reader :keys, :record_count
+    attr_reader :keys, :record_count, :layout
 
     def initialize(bytes)
       @data = bytes.b # force binary encoding
@@ -51,12 +61,14 @@ module Nxs
       raise NxsError.new('ERR_BAD_MAGIC', 'footer magic mismatch') if footer != MAGIC_FOOTER
 
       # Preamble: Version(2) + Flags(2) + DictHash(8) + TailPtr(8) + Reserved(8)
-      @flags    = @data.unpack1('@6 S<')
-      @tail_ptr = @data.unpack1('@16 Q<')
-      if @tail_ptr.zero?
+      @flags         = @data.unpack1('@6 S<')
+      preamble_tail  = @data.unpack1('@16 Q<')
+      @tail_ptr       = preamble_tail
+      layout_flags    = @flags & (FLAG_COLUMNAR | FLAG_PAX)
+      if @tail_ptr.zero? && layout_flags.zero?
         raise NxsError.new('ERR_OUT_OF_BOUNDS', 'stream footer') if sz < 44
 
-        @tail_ptr = @data.unpack1("@#{sz - 12}Q<")
+        @tail_ptr = @data.unpack1("@#{sz - FOOTER_ROW_BYTES}Q<")
       end
 
       @dict_hash = @data.unpack1('@8 Q<')
@@ -71,24 +83,27 @@ module Nxs
         raise NxsError.new('ERR_DICT_MISMATCH', 'schema hash mismatch') if computed != @dict_hash
       end
 
-      # Tail-index: u32 EntryCount followed by records
-      @record_count = @data.unpack1("@#{@tail_ptr}L<")
-      @tail_start   = @tail_ptr + 4
+      @col_buf_off = []
+      @col_buf_len = []
+      parse_layout_tail!(preamble_tail)
     end
 
-    # O(1) record lookup — reads one 10-byte tail-index entry.
+    # O(1) record lookup — row tail-index or columnar/PAX record index.
     def record(i)
       unless i >= 0 && i < @record_count
         raise NxsError.new('ERR_OUT_OF_BOUNDS', "record #{i} out of [0, #{@record_count})")
       end
 
-      # Each tail-index entry: u16 KeyID + u64 AbsoluteOffset = 10 bytes
+      return Object.new(self, i, i) if @layout != :row
+
       abs_offset = @data.unpack1("@#{@tail_start + i * 10 + 2}Q<")
       Object.new(self, abs_offset)
     end
 
-    # Tight allocation-free sum loop.
+    # Sum f64 column — columnar/PAX buffer path or row scan.
     def sum_f64(key)
+      return col_sum_f64(key) if @layout != :row
+
       slot = @key_index[key]
       raise NxsError.new('ERR_OUT_OF_BOUNDS', "key '#{key}' not in schema") unless slot
 
@@ -106,6 +121,68 @@ module Nxs
       sum
     end
 
+    # Columnar/PAX f64 sum (row layout delegates to sum_f64).
+    def col_sum_f64(key)
+      slot = @key_index[key]
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', "key '#{key}' not in schema") unless slot
+
+      return sum_f64(key) if @layout == :row
+      return pax_sum_f64(slot) if @layout == :pax
+
+      bm, vals = col_field_parts(slot)
+      n = @record_count
+      sum = 0.0
+      i = 0
+      while i < n
+        if col_bit(bm, i)
+          off = i * 8
+          sum += vals.unpack1("@#{off}E") if off + 8 <= vals.bytesize
+        end
+        i += 1
+      end
+      sum
+    end
+
+    # Raw value bytes for a fixed-width column (columnar/PAX).
+    def col_buffer(key)
+      raise NxsError.new('ERR_LAYOUT', 'col_buffer requires columnar or PAX layout') if @layout == :row
+
+      slot = @key_index[key]
+      return nil unless slot
+      return nil if var_sigil?(@key_sigils[slot])
+
+      _bm, vals = col_field_parts(slot)
+      vals
+    rescue NxsError
+      nil
+    end
+
+    # Null bitmap + u32 offsets + values for var-length columns (columnar only).
+    def col_var_buffer(key)
+      raise NxsError.new('ERR_LAYOUT', 'col_var_buffer is columnar-only') unless @layout == :columnar
+
+      slot = @key_index[key]
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', "key '#{key}' not in schema") unless slot
+      raise NxsError.new('ERR_UNSUPPORTED_FIELD_TYPE', key) unless var_sigil?(@key_sigils[slot])
+
+      bm, offsets, values = col_var_parts(slot)
+      { bitmap: bm, offsets: offsets, values: values, count: @record_count }
+    end
+
+    def col_get_str(key, record_index)
+      slot = @key_index[key]
+      return nil unless slot && record_index < @record_count && @layout != :row
+      return nil unless @key_sigils[slot] == 0x22
+
+      bm, offsets, values, ok = col_var_parts_at(record_index, slot)
+      return nil unless ok
+
+      bit_idx = @layout == :pax ? pax_find_page(record_index)&.[](:local) : record_index
+      return nil if bit_idx.nil? || !col_bit(bm, bit_idx)
+
+      var_str_at(offsets, values, bit_idx)
+    end
+
     def min_f64(key)
       slot = @key_index[key]
       raise NxsError.new('ERR_OUT_OF_BOUNDS', "key '#{key}' not in schema") unless slot
@@ -178,6 +255,8 @@ module Nxs
       t_idx = 0
 
       loop do
+        raise NxsError.new('ERR_OUT_OF_BOUNDS', 'bitmask overrun on corrupt input') if p >= data.bytesize
+
         b    = data.getbyte(p)
         p   += 1
         bits = b & 0x7F
@@ -205,6 +284,285 @@ module Nxs
 
     private
 
+    def parse_layout_tail!(preamble_tail)
+      if (@flags & FLAG_COLUMNAR != 0) && (@flags & FLAG_PAX != 0)
+        raise NxsError.new('ERR_INVALID_FLAGS', 'columnar and PAX both set')
+      end
+      if (@flags & FLAG_COLUMNAR != 0) && preamble_tail.zero?
+        raise NxsError.new('ERR_INCOMPATIBLE_FLAGS', 'columnar with TailPtr=0')
+      end
+
+      if (@flags & FLAG_COLUMNAR) != 0
+        @layout = :columnar
+        parse_columnar_footer!
+        return
+      end
+      if (@flags & FLAG_PAX) != 0
+        @layout = :pax
+        parse_pax_footer!
+        return
+      end
+
+      @layout = :row
+      if preamble_tail.zero?
+        raise NxsError.new('ERR_OUT_OF_BOUNDS', 'streamable footer') if @data.bytesize < 44
+
+        @tail_ptr = @data.unpack1("@#{@data.bytesize - FOOTER_ROW_BYTES}Q<")
+      end
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'tail index') if @tail_ptr + 4 > @data.bytesize
+
+      @record_count = @data.unpack1("@#{@tail_ptr}L<")
+      @tail_start   = @tail_ptr + 4
+    end
+
+    def parse_columnar_footer!
+      sz = @data.bytesize
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'columnar footer') if sz < FOOTER_COL_BYTES
+
+      fo = sz - FOOTER_COL_BYTES
+      @tail_ptr      = @data.unpack1("@#{fo}Q<")
+      @record_count  = @data.unpack1("@#{fo + 8}Q<")
+      @tail_start    = @tail_ptr
+      kc = @keys.length
+      @col_buf_off = Array.new(kc)
+      @col_buf_len = Array.new(kc)
+      kc.times do |i|
+        e = @tail_start + i * COL_TAIL_ENTRY_BYTES
+        raise NxsError.new('ERR_OUT_OF_BOUNDS', 'columnar tail entry') if e + COL_TAIL_ENTRY_BYTES > sz
+
+        fid = @data.unpack1("@#{e}S<")
+        raise NxsError.new('ERR_OUT_OF_BOUNDS', "invalid field ID #{fid}") if fid >= kc
+
+        @col_buf_off[fid] = @data.unpack1("@#{e + 4}Q<")
+        @col_buf_len[fid] = @data.unpack1("@#{e + 12}Q<")
+      end
+    end
+
+    def parse_pax_footer!
+      sz = @data.bytesize
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'PAX footer') if sz < FOOTER_PAX_BYTES
+
+      fo = sz - FOOTER_PAX_BYTES
+      @tail_ptr       = @data.unpack1("@#{fo}Q<")
+      @record_count   = @data.unpack1("@#{fo + 8}Q<")
+      @page_count     = @data.unpack1("@#{fo + 16}L<")
+      @page_size_hint = @data.unpack1("@#{fo + 20}L<")
+      @tail_start     = @tail_ptr
+      @page_index     = []
+      @page_rec_start = []
+      @page_rec_count = []
+      @page_offset    = []
+      @page_length    = []
+
+      @page_count.times do |i|
+        e = @tail_start + i * PAX_TAIL_ENTRY_BYTES
+        raise NxsError.new('ERR_OUT_OF_BOUNDS', 'PAX tail entry') if e + PAX_TAIL_ENTRY_BYTES > sz
+
+        @page_index << @data.unpack1("@#{e}L<")
+        @page_rec_start << @data.unpack1("@#{e + 4}Q<")
+        @page_rec_count << @data.unpack1("@#{e + 12}L<")
+        @page_offset << @data.unpack1("@#{e + 16}Q<")
+        @page_length << @data.unpack1("@#{e + 24}L<")
+      end
+
+      @page_count.times do |i|
+        poff = @page_offset[i]
+        if poff > sz || poff + 4 > sz || @data.unpack1("@#{poff}L<") != MAGIC_PAGE
+          raise NxsError.new('ERR_INVALID_PAGE_MAGIC', 'PAX page magic mismatch')
+        end
+      end
+    end
+
+    def null_bitmap_bytes(n)
+      raw = (n + 7) / 8
+      (raw + 7) & ~7
+    end
+
+    # rubocop:disable Naming/PredicateMethod -- mirrors C col_bit naming
+    def col_bit(bm, rec)
+      ((bm.getbyte(rec / 8) >> (rec % 8)) & 1) == 1
+    end
+    # rubocop:enable Naming/PredicateMethod
+
+    def var_sigil?(sig)
+      [0x22, 0x3C].include?(sig)
+    end
+
+    def var_off_bytes_len(rc)
+      off = (rc + 1) * 4
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'var offsets overflow') if off > @data.bytesize
+
+      off
+    end
+
+    def field_sector_len(sector_off, rc, sigil)
+      bm_len = null_bitmap_bytes(rc)
+      return bm_len + rc * 8 unless var_sigil?(sigil)
+
+      off_bytes = var_off_bytes_len(rc)
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'var offsets') if sector_off + bm_len + off_bytes > @data.bytesize
+
+      end_off = @data.unpack1("@#{sector_off + bm_len + rc * 4}L<")
+      total = bm_len + off_bytes + end_off
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'var values') if sector_off + total > @data.bytesize
+
+      total
+    end
+
+    def var_str_at(offsets, values, record_index)
+      need = (record_index + 2) * 4
+      return nil if offsets.bytesize < need
+
+      off = record_index * 4
+      start = offsets.unpack1("@#{off}L<")
+      end_  = offsets.unpack1("@#{off + 4}L<")
+      return nil if end_ < start || end_ > values.bytesize
+
+      values[start...end_].force_encoding('UTF-8')
+    end
+
+    def col_field_parts(slot)
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', "key slot #{slot}") if slot.negative? || slot >= @col_buf_off.length
+
+      off = @col_buf_off[slot].to_i
+      length = @col_buf_len[slot].to_i
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'column buffer') if off + length > @data.bytesize
+
+      bm_len = null_bitmap_bytes(@record_count)
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'null bitmap') if length < bm_len
+
+      sector = @data[off, length]
+      [sector[0, bm_len], sector[bm_len..]]
+    end
+
+    def col_var_parts(slot)
+      bm, tail = col_field_parts(slot)
+      off_bytes = var_off_bytes_len(@record_count)
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'var offsets') if tail.bytesize < off_bytes
+
+      [bm, tail[0, off_bytes], tail[off_bytes..]]
+    end
+
+    def col_var_parts_at(rec, slot)
+      return [nil, nil, nil, false] if slot.negative? || slot >= @key_sigils.length || !var_sigil?(@key_sigils[slot])
+
+      if @layout == :columnar
+        bm, offsets, values = col_var_parts(slot)
+        return [bm, offsets, values, true]
+      end
+      if @layout == :pax
+        loc = pax_find_page(rec)
+        return [nil, nil, nil, false] unless loc
+
+        bm, tail = page_field_parts(loc[:page], slot)
+        return [nil, nil, nil, false] unless bm
+
+        rc = @page_rec_count[loc[:page]]
+        off_bytes = var_off_bytes_len(rc)
+        return [nil, nil, nil, false] if tail.bytesize < off_bytes
+
+        return [bm, tail[0, off_bytes], tail[off_bytes..], true]
+      end
+      [nil, nil, nil, false]
+    end
+
+    def col_numeric_bytes(rec, slot)
+      return nil if slot >= 0 && slot < @key_sigils.length && var_sigil?(@key_sigils[slot])
+
+      if @layout == :columnar
+        bm, vals = col_field_parts(slot)
+        return nil if rec >= @record_count || !col_bit(bm, rec)
+
+        off = rec * 8
+        return nil if off + 8 > vals.bytesize
+
+        return vals[off, 8]
+      end
+      if @layout == :pax
+        loc = pax_find_page(rec)
+        return nil unless loc
+
+        bm, vals = page_field_parts(loc[:page], slot)
+        return nil unless bm && col_bit(bm, loc[:local])
+
+        off = loc[:local] * 8
+        return nil if off + 8 > vals.bytesize
+
+        return vals[off, 8]
+      end
+      nil
+    end
+
+    def pax_find_page(rec)
+      return nil if @page_count.zero?
+
+      lo = 0
+      hi = @page_count - 1
+      while lo <= hi
+        mid = lo + (hi - lo) / 2
+        start = @page_rec_start[mid]
+        count = @page_rec_count[mid]
+        if rec < start
+          hi = mid - 1
+        elsif rec >= start + count
+          lo = mid + 1
+        else
+          return { page: mid, local: rec - start }
+        end
+      end
+      nil
+    end
+
+    def page_field_sector(pi, slot)
+      poff = @page_offset[pi].to_i
+      return nil if poff + 24 > @data.bytesize || @data.unpack1("@#{poff}L<") != MAGIC_PAGE
+
+      fc = @data.unpack1("@#{poff + 20}S<")
+      return nil if slot.negative? || slot >= fc || fc > @key_sigils.length
+
+      rc = @page_rec_count[pi]
+      body = poff + 24
+      slot.times do |fi|
+        sig = fi < @key_sigils.length ? @key_sigils[fi] : 0x3D
+        flen = field_sector_len(body, rc, sig)
+        body += flen
+      end
+      sig = slot < @key_sigils.length ? @key_sigils[slot] : 0x3D
+      flen = field_sector_len(body, rc, sig)
+      return nil if body + flen > @data.bytesize
+
+      @data[body, flen]
+    end
+
+    def page_field_parts(pi, slot)
+      sector = page_field_sector(pi, slot)
+      return [nil, nil] unless sector
+
+      bm_len = null_bitmap_bytes(@page_rec_count[pi])
+      return [nil, nil] if sector.bytesize < bm_len
+
+      [sector[0, bm_len], sector[bm_len..]]
+    end
+
+    def pax_sum_f64(slot)
+      sum = 0.0
+      @page_count.times do |pi|
+        bm, vals = page_field_parts(pi, slot)
+        next unless bm
+
+        rc = @page_rec_count[pi]
+        i = 0
+        while i < rc
+          if col_bit(bm, i)
+            off = i * 8
+            sum += vals.unpack1("@#{off}E") if off + 8 <= vals.bytesize
+          end
+          i += 1
+        end
+      end
+      sum
+    end
+
     def read_schema(offset)
       key_count = @data.unpack1("@#{offset}S<")
       offset += 2
@@ -438,13 +796,19 @@ module Nxs
   # ── Object ───────────────────────────────────────────────────────────────────
 
   class Object
-    def initialize(reader, offset)
-      @reader = reader
-      @offset = offset
-      @parsed = false
+    def initialize(reader, offset, record_index = nil)
+      @reader       = reader
+      @offset       = offset
+      @record_index = record_index
+      @parsed       = false
     end
 
     def get_str(key)
+      slot = @reader.key_index[key]
+      return nil unless slot
+
+      return @reader.col_get_str(key, record_index) if uses_columnar_field_access?
+
       off = field_offset(key)
       return nil unless off
 
@@ -453,6 +817,16 @@ module Nxs
     end
 
     def get_i64(key)
+      slot = @reader.key_index[key]
+      return nil unless slot
+
+      if uses_columnar_field_access?
+        cell = @reader.send(:col_numeric_bytes, record_index, slot)
+        return nil unless cell
+
+        return cell.unpack1('q<')
+      end
+
       off = field_offset(key)
       return nil unless off
 
@@ -460,6 +834,16 @@ module Nxs
     end
 
     def get_f64(key)
+      slot = @reader.key_index[key]
+      return nil unless slot
+
+      if uses_columnar_field_access?
+        cell = @reader.send(:col_numeric_bytes, record_index, slot)
+        return nil unless cell
+
+        return cell.unpack1('E')
+      end
+
       off = field_offset(key)
       return nil unless off
 
@@ -467,6 +851,16 @@ module Nxs
     end
 
     def get_bool(key)
+      slot = @reader.key_index[key]
+      return nil unless slot
+
+      if uses_columnar_field_access?
+        cell = @reader.send(:col_numeric_bytes, record_index, slot)
+        return nil unless cell
+
+        return cell.getbyte(0) != 0
+      end
+
       off = field_offset(key)
       return nil unless off
 
@@ -475,6 +869,21 @@ module Nxs
 
     private
 
+    def record_index
+      @record_index.nil? ? @offset : @record_index
+    end
+
+    def obj_at_nyxo?
+      return false if @offset + 4 > @reader.data.bytesize
+
+      @reader.data.unpack1("@#{@offset}L<") == MAGIC_OBJ
+    end
+
+    # Columnar/PAX top-level records use record index; nested NYXO blobs use row paths.
+    def uses_columnar_field_access?
+      @reader.layout != :row && !obj_at_nyxo?
+    end
+
     # Parse the object header (lazy — only on first field access).
     def parse_header
       return if @parsed

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nyxis
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Micael Malta
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-05-20 00:00:00.000000000 Z
+date: 2026-05-22 00:00:00.000000000 Z
 dependencies: []
 description: |
   Pure-Ruby reader for NXB files produced by the NXS compiler. Provides