nexus-standard 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 995c29cd7138de91cb54cc1466b42115597d2575fb0d4422e0ab8020a3f42817
+  data.tar.gz: 93514584b431ceead1fdfbdb59332657b134dfd1351c944776c0b24c85ab7103
+SHA512:
+  metadata.gz: d9cef581fd5964f14dada1b0230c180c0085ee2b0844ba5639794e3ca1a1c22a6482b21edd07733095a6fe93cbb1bbaf3f53308e1982c53625b9bf1a89ba1203
+  data.tar.gz: f958845efc6bc1be795aef861acb578cfdc7c67f38ac886e4c84a2fa71d0335f3347e533a702b9b1ba08ea55cb0172e8ecfcf6ad505e49aca9c4f195474d643b

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Micael Malta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,101 @@
+# NXS — Ruby
+
+Zero-copy `.nxb` reader for Ruby 3.x. Pure-Ruby implementation with an optional C extension for hot-path columnar scans. No gems required.
+
+## Requirements
+
+Ruby 3.0+. The C extension requires a C compiler and Ruby headers (`ruby-dev` / `ruby-devel`).
+
+## Read a file
+
+```ruby
+require_relative "nxs"
+
+bytes  = File.binread("data.nxb")
+reader = Nxs::Reader.new(bytes)
+
+puts reader.record_count              # instant — read from tail-index, no parse pass
+obj = reader.record(42)               # O(1) seek
+puts obj.get_str("username")
+puts obj.get_f64("score")
+puts obj.get_bool("active")
+puts obj.get_i64("id")
+```
+
+## Columnar reducers
+
+```ruby
+total = reader.sum_f64("score")
+low   = reader.min_f64("score")
+high  = reader.max_f64("score")
+ages  = reader.sum_i64("age")
+```
+
+## C extension (hot path)
+
+Build once:
+
+```bash
+bash ext/build.sh
+```
+
+```ruby
+require_relative "ext/nxs/nxs_ext"   # loads Nxs::CReader and Nxs::CObject
+
+reader = Nxs::CReader.new(bytes)
+puts reader.record(42).get_str("username")
+puts reader.sum_f64("score")          # 6.78 ms at 1M records vs 942 ms pure Ruby
+```
+
+At 1M records the C extension is **139× faster** than pure Ruby for `sum_f64`, and **5.6× faster** than `JSON.parse`.
+
+## Write a file
+
+```ruby
+require_relative "nxs_writer"
+
+schema = Nxs::Schema.new(["id", "username", "score", "active"])
+w = Nxs::Writer.new(schema)
+
+w.begin_object
+w.write_i64(0, 42)
+w.write_str(1, "alice")
+w.write_f64(2, 9.5)
+w.write_bool(3, true)
+w.end_object
+
+data = w.finish   # binary String (encoding ASCII-8BIT)
+
+# Convenience: write from an array of hashes
+data2 = Nxs::Writer.from_records(
+  ["id", "username", "score"],
+  [{ "id" => 1, "username" => "bob", "score" => 8.2 }]
+)
+```
+
+## Tests
+
+```bash
+ruby test.rb ../js/fixtures    # 22 tests
+```
+
+## Benchmarks
+
+```bash
+ruby bench.rb ../js/fixtures       # pure Ruby vs JSON
+ruby bench_c.rb ../js/fixtures     # C extension vs JSON
+```
+
+## Files
+
+| File | Purpose |
+| :--- | :--- |
+| `nxs.rb` | Pure-Ruby reader (`Nxs::Reader`, `Nxs::Object`) |
+| `nxs_writer.rb` | Pure-Ruby writer (`Nxs::Schema`, `Nxs::Writer`) |
+| `ext/nxs/nxs_ext.c` | C extension source (`Nxs::CReader`, `Nxs::CObject`) |
+| `ext/nxs/extconf.rb` | Extension build configuration |
+| `ext/build.sh` | Compiles the C extension |
+
+---
+
+For the format specification see [`SPEC.md`](../SPEC.md). For cross-language examples see [`GETTING_STARTED.md`](../GETTING_STARTED.md).

data/nxs.rb

@@ -0,0 +1,327 @@
+# frozen_string_literal: true
+
+# NXS Reader — .nxb parser (Ruby 3.x, stdlib only).
+#
+# Implements Nexus Standard v1.0 binary wire format.
+#
+# Usage:
+#   buf = File.binread("data.nxb")
+#   reader = Nxs::Reader.new(buf)
+#   reader.record_count          # => Integer
+#   reader.keys                  # => Array<String>
+#   obj = reader.record(42)      # => Nxs::Object
+#   obj.get_str("username")      # => String | nil
+#   obj.get_i64("id")            # => Integer | nil
+#   obj.get_f64("score")         # => Float | nil
+#   obj.get_bool("active")       # => true/false | nil
+#   reader.sum_f64("score")      # => Float
+#   reader.min_f64("score")      # => Float | nil
+#   reader.max_f64("score")      # => Float | nil
+#   reader.sum_i64("id")         # => Integer
+
+module Nxs
+  MAGIC_FILE   = 0x4E585342  # NXSB
+  MAGIC_OBJ    = 0x4E58534F  # NXSO
+  MAGIC_FOOTER = 0x2153584E  # NXS!
+  FLAG_SCHEMA  = 0x0002
+
+  class NxsError < StandardError
+    attr_reader :code
+
+    def initialize(code, msg)
+      super("#{code}: #{msg}")
+      @code = code
+    end
+  end
+
+  # ── Reader ──────────────────────────────────────────────────────────────────
+
+  class Reader
+    attr_reader :keys, :record_count
+
+    def initialize(bytes)
+      @data = bytes.b # force binary encoding
+      sz = @data.bytesize
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', 'file too small') if sz < 32
+
+      magic = @data.unpack1('L<')
+      raise NxsError.new('ERR_BAD_MAGIC', "expected NXSB, got 0x#{magic.to_s(16)}") if magic != MAGIC_FILE
+
+      footer = @data.unpack1("@#{sz - 4}L<")
+      raise NxsError.new('ERR_BAD_MAGIC', 'footer magic mismatch') if footer != MAGIC_FOOTER
+
+      # Preamble: Version(2) + Flags(2) + DictHash(8) + TailPtr(8) + Reserved(8)
+      @flags    = @data.unpack1('@6 S<')
+      @tail_ptr = @data.unpack1('@16 Q<')
+
+      @dict_hash = @data.unpack1('@8 Q<')
+
+      # Schema (when Flags bit 1 set)
+      @keys       = []
+      @key_sigils = []
+      @key_index  = {}
+      if @flags & FLAG_SCHEMA != 0
+        schema_end = read_schema(32)
+        computed = murmur3_64(@data[32...schema_end].bytes)
+        raise NxsError.new('ERR_DICT_MISMATCH', 'schema hash mismatch') if computed != @dict_hash
+      end
+
+      # Tail-index: u32 EntryCount followed by records
+      @record_count = @data.unpack1("@#{@tail_ptr}L<")
+      @tail_start   = @tail_ptr + 4
+    end
+
+    # O(1) record lookup — reads one 10-byte tail-index entry.
+    def record(i)
+      unless i >= 0 && i < @record_count
+        raise NxsError.new('ERR_OUT_OF_BOUNDS', "record #{i} out of [0, #{@record_count})")
+      end
+
+      # Each tail-index entry: u16 KeyID + u64 AbsoluteOffset = 10 bytes
+      abs_offset = @data.unpack1("@#{@tail_start + i * 10 + 2}Q<")
+      Object.new(self, abs_offset)
+    end
+
+    # Tight allocation-free sum loop.
+    def sum_f64(key)
+      slot = @key_index[key]
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', "key '#{key}' not in schema") unless slot
+
+      data = @data
+      tail = @tail_start
+      n    = @record_count
+      sum  = 0.0
+      i = 0
+      while i < n
+        abs = data.unpack1("@#{tail + i * 10 + 2}Q<")
+        off = _scan_offset(data, abs, slot)
+        sum += data.unpack1("@#{off}E") if off
+        i += 1
+      end
+      sum
+    end
+
+    def min_f64(key)
+      slot = @key_index[key]
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', "key '#{key}' not in schema") unless slot
+
+      data = @data
+      tail = @tail_start
+      n    = @record_count
+      min  = nil
+      i = 0
+      while i < n
+        abs = data.unpack1("@#{tail + i * 10 + 2}Q<")
+        off = _scan_offset(data, abs, slot)
+        if off
+          v   = data.unpack1("@#{off}E")
+          min = v if min.nil? || v < min
+        end
+        i += 1
+      end
+      min
+    end
+
+    def max_f64(key)
+      slot = @key_index[key]
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', "key '#{key}' not in schema") unless slot
+
+      data = @data
+      tail = @tail_start
+      n    = @record_count
+      max  = nil
+      i = 0
+      while i < n
+        abs = data.unpack1("@#{tail + i * 10 + 2}Q<")
+        off = _scan_offset(data, abs, slot)
+        if off
+          v   = data.unpack1("@#{off}E")
+          max = v if max.nil? || v > max
+        end
+        i += 1
+      end
+      max
+    end
+
+    def sum_i64(key)
+      slot = @key_index[key]
+      raise NxsError.new('ERR_OUT_OF_BOUNDS', "key '#{key}' not in schema") unless slot
+
+      data = @data
+      tail = @tail_start
+      n    = @record_count
+      sum  = 0
+      i = 0
+      while i < n
+        abs = data.unpack1("@#{tail + i * 10 + 2}Q<")
+        off = _scan_offset(data, abs, slot)
+        sum += data.unpack1("@#{off}q<") if off
+        i += 1
+      end
+      sum
+    end
+
+    # Expose internals for Object
+    attr_reader :data, :key_index
+
+    # Walk the LEB128 bitmask from obj_offset+8, count set bits before `slot`,
+    # and return the absolute byte offset of the field value (or nil if absent).
+    # Used by both bulk reducers and NxsObject.
+    def _scan_offset(data, obj_offset, slot)
+      p = obj_offset + 8 # skip Magic(4) + Length(4)
+      cur   = 0
+      t_idx = 0
+
+      loop do
+        b    = data.getbyte(p)
+        p   += 1
+        bits = b & 0x7F
+        7.times do |i|
+          if cur == slot
+            # field absent if bit is 0
+            return nil if ((bits >> i) & 1).zero?
+
+            # p already past this bitmask byte; drain remaining continuation bytes
+            while (b & 0x80) != 0
+              b = data.getbyte(p)
+              p += 1
+            end
+            # p now points to the offset table
+            rel = data.unpack1("@#{p + t_idx * 2}S<")
+            return obj_offset + rel
+          end
+          t_idx += 1 if (bits >> i) & 1 == 1
+          cur += 1
+        end
+        # If all 7 bits processed and continuation bit clear, field is absent
+        return nil if (b & 0x80).zero?
+      end
+    end
+
+    private
+
+    def read_schema(offset)
+      key_count = @data.unpack1("@#{offset}S<")
+      offset += 2
+
+      @key_sigils = @data[offset, key_count].bytes
+      offset += key_count
+
+      # Null-terminated UTF-8 strings in StringPool
+      pool = @data[offset..]
+      pos  = 0
+      key_count.times do |i|
+        term = pool.index("\x00", pos)
+        @keys << pool[pos...term].force_encoding('UTF-8')
+        @key_index[@keys.last] = i
+        pos = term + 1
+      end
+      offset += pos
+
+      # Pad to 8-byte boundary
+      rem = offset % 8
+      offset += (8 - rem) % 8
+      offset
+    end
+
+    MURMUR_C1 = 0xFF51AFD7ED558CCD
+    MURMUR_C2 = 0xC4CEB9FE1A85EC53
+    MURMUR_MASK = 0xFFFFFFFFFFFFFFFF
+
+    def murmur3_64(bytes)
+      h = 0x93681D6255313A99
+      i = 0
+      len = bytes.length
+      while i < len
+        chunk = bytes[i, 8]
+        k = 0
+        chunk.each_with_index { |b, j| k |= b << (j * 8) }
+        k = (k * MURMUR_C1) & MURMUR_MASK
+        k ^= k >> 33
+        h ^= k
+        h = (h * MURMUR_C2) & MURMUR_MASK
+        h ^= h >> 33
+        i += 8
+      end
+      h ^= len
+      h ^= h >> 33
+      h = (h * MURMUR_C1) & MURMUR_MASK
+      h ^= h >> 33
+      h
+    end
+  end
+
+  # ── Object ───────────────────────────────────────────────────────────────────
+
+  class Object
+    def initialize(reader, offset)
+      @reader = reader
+      @offset = offset
+      @parsed = false
+    end
+
+    def get_str(key)
+      off = field_offset(key)
+      return nil unless off
+
+      len = @reader.data.unpack1("@#{off}L<")
+      @reader.data[off + 4, len].force_encoding('UTF-8')
+    end
+
+    def get_i64(key)
+      off = field_offset(key)
+      return nil unless off
+
+      @reader.data.unpack1("@#{off}q<")
+    end
+
+    def get_f64(key)
+      off = field_offset(key)
+      return nil unless off
+
+      @reader.data.unpack1("@#{off}E")
+    end
+
+    def get_bool(key)
+      off = field_offset(key)
+      return nil unless off
+
+      @reader.data.getbyte(off) != 0
+    end
+
+    private
+
+    # Parse the object header (lazy — only on first field access).
+    def parse_header
+      return if @parsed
+
+      p = @offset
+
+      magic = @reader.data.unpack1("@#{p}L<")
+      raise NxsError.new('ERR_BAD_MAGIC', "expected NXSO at #{p}") if magic != MAGIC_OBJ
+
+      p += 8 # skip Magic(4) + Length(4)
+
+      bitmask = []
+      loop do
+        b = @reader.data.getbyte(p)
+        p += 1
+        bitmask << (b & 0x7F)
+        break if (b & 0x80).zero?
+      end
+
+      @bitmask          = bitmask
+      @offset_tbl_start = p
+      @parsed           = true
+    end
+
+    # Return the absolute byte offset of the field for `key`, or nil.
+    def field_offset(key)
+      slot = @reader.key_index[key]
+      return nil unless slot
+
+      # Delegate to Reader's scan logic (same implementation, avoids duplication)
+      @reader._scan_offset(@reader.data, @offset, slot)
+    end
+  end
+end

metadata

@@ -0,0 +1,51 @@
+--- !ruby/object:Gem::Specification
+name: nexus-standard
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Micael Malta
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-05-01 00:00:00.000000000 Z
+dependencies: []
+description: |
+  Pure-Ruby reader for NXB files produced by the NXS compiler. Provides
+  zero-copy memory-mapped access to typed records with O(1) random access
+  via the tail-index.
+email:
+- micael@example.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- nxs.rb
+homepage: https://github.com/micaelmalta/nexus-standard
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/micaelmalta/nexus-standard
+  changelog_uri: https://github.com/micaelmalta/nexus-standard/releases
+post_install_message:
+rdoc_options: []
+require_paths:
+- "."
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Zero-copy reader for the Nexus Standard (NXS) binary format
+test_files: []