igniter-ledger 0.5.2

data/lib/igniter/store/codecs.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require "json"
+require "zlib"
+require "digest"
+require "securerandom"
+require "msgpack"
+require_relative "wire_protocol"
+
+module Igniter
+  module Store
+    # Pluggable per-segment codec system for SegmentedFileBackend.
+    #
+    # Each codec owns the write side (how a Fact becomes bytes in a segment)
+    # and the read side (how segment bytes become Facts on replay).
+    #
+    # Codec lifecycle per segment:
+    #
+    #   codec = Codecs.build(:compact_delta)
+    #   codec.start_segment(io, store: "readings")   # optional header frame
+    #   codec.encode_fact(io, fact)                  # returns bytes written
+    #   ...
+    #   codec.flush(io)                              # flush any buffered data
+    #   ── seal ──
+    #
+    #   codec2 = Codecs.build(:compact_delta)
+    #   facts  = codec2.decode(io)                   # reads whole segment
+    #
+    # Codec instances are stateful and single-use per segment.
+    module Codecs
+      # Build a fresh codec instance by name.
+      def self.build(name)
+        case name.to_sym
+        when :json_crc32
+          JsonCrc32.new
+        when :compact_delta, :"compact_delta_zlib"
+          CompactDelta.new
+        else
+          raise ArgumentError, "Unknown codec: #{name.inspect}"
+        end
+      end
+
+      # ── JsonCrc32 ───────────────────────────────────────────────────────────
+      #
+      # One CRC32-framed JSON frame per fact. Matches the pure-Ruby FileBackend
+      # format — readable without any extra dependencies.
+      class JsonCrc32
+        include WireProtocol
+
+        NAME = "json_crc32"
+
+        def name = NAME
+
+        def start_segment(_io, store: nil) = 0   # no header needed
+
+        def encode_fact(io, fact)
+          frame = encode_frame(JSON.generate(fact.to_h))
+          io.write(frame)
+        end
+
+        def flush(_io) = 0   # stateless, nothing buffered
+
+        def buffered_count = 0
+
+        def decode(io)
+          facts = []
+          loop do
+            body = read_frame(io)
+            break unless body
+            fact = Fact.from_h(JSON.parse(body, symbolize_names: true)) rescue nil
+            facts << fact if fact
+          end
+          facts
+        end
+      end
+
+      # ── CompactDelta ────────────────────────────────────────────────────────
+      #
+      # Structural compression optimised for high-frequency History stores
+      # (sensor readings, GPS tracks, telemetry).
+      #
+      # What is removed vs the full Fact representation:
+      #   id            → not stored; synthetic id assigned on decode
+      #   store         → in segment header (once per segment)
+      #   value_hash    → not stored; recomputed from value on decode
+      #   causation     → always nil for History — omitted
+      #   term          → in segment header
+      #   schema_version→ in segment header
+      #   value keys    → field index from per-segment dictionary (header frame)
+      #   key string    → per-segment key dictionary index (delta updates per batch)
+      #   timestamp     → absolute ms for first entry; signed delta-ms thereafter
+      #
+      # Segment layout (all frames use WireProtocol CRC32 framing):
+      #   [header_frame]  MessagePack { store, fields:[...], term, schema_version }
+      #   [batch_frame]   MessagePack { km:{idx=>key,...}, e:[[ki,Δms,[v0,v1…]],…] }
+      #                   compressed with Zlib before framing
+      #   ...
+      #
+      # The key map (km) in each batch carries only NEW keys added since the
+      # previous batch, so readers accumulate it incrementally.
+      #
+      # Benchmark result (GPS stream, 5 k facts):
+      #   json_crc32   → 380 bytes/fact
+      #   compact_delta→  23 bytes/fact   (16x smaller)
+      #
+      # Limitation (native mode): decoded Facts receive synthetic ids and, in
+      # the Rust native extension, timestamps are reset to Time.now — the same
+      # known gap as json_crc32 in native mode.  Pure-Ruby mode restores
+      # timestamps correctly.
+      class CompactDelta
+        include WireProtocol
+
+        NAME       = "compact_delta_zlib"
+        BATCH_SIZE = 64
+
+        def name = NAME
+
+        # ── Write side ──────────────────────────────────────────────────────
+
+        def initialize
+          @fields          = nil
+          @key_map         = {}   # key_string → Integer index
+          @km_flushed      = 0    # keys already sent to disk
+          @last_ts_ms      = nil
+          @batch_buf       = []
+          @header_written  = false
+          @store           = nil
+        end
+
+        def start_segment(_io, store: nil)
+          @store = store.to_s
+          0  # header is written lazily on first encode_fact call
+        end
+
+        # Returns bytes written to io (0 while batch is buffered).
+        def encode_fact(io, fact)
+          unless @header_written
+            @fields = (fact.value || {}).keys.map(&:to_s).sort
+            header  = { store: fact.store.to_s, fields: @fields,
+                        valid_time: fact.valid_time, schema_version: fact.schema_version }
+            body = MessagePack.pack(stringify(header))
+            io.write(encode_frame(body))
+            @header_written = true
+          end
+
+          @batch_buf << fact
+          @batch_buf.size >= BATCH_SIZE ? write_batch(io) : 0
+        end
+
+        # Flush any remaining buffered facts. Returns bytes written.
+        def flush(io)
+          @batch_buf.empty? ? 0 : write_batch(io)
+        end
+
+        def buffered_count = @batch_buf.size
+
+        # ── Read side ────────────────────────────────────────────────────────
+
+        def decode(io)
+          header_body = read_frame(io)
+          return [] unless header_body
+          header = MessagePack.unpack(header_body)
+          fields = header["fields"] || []
+          store      = (header["store"] || "").to_sym
+          valid_time = (header["valid_time"] || header["term"])&.to_f
+          sv         = (header["schema_version"] || 1).to_i
+
+          key_map    = {}    # Integer index → key_string
+          last_ts_ms = nil
+          facts      = []
+
+          while (body = read_frame(io))
+            _count = body[0, 4].unpack1("N")
+            raw    = Zlib::Inflate.inflate(body[4..])
+            batch  = MessagePack.unpack(raw)
+
+            (batch["km"] || {}).each { |idx, key| key_map[idx.to_i] = key }
+
+            (batch["e"] || []).each do |key_idx, delta_ms, vals|
+              ts_ms      = last_ts_ms ? last_ts_ms + delta_ms : delta_ms
+              last_ts_ms = ts_ms
+
+              value = fields.each_with_index.to_h { |fn, i| [fn.to_sym, vals[i]] }
+              vh    = Digest::SHA256.hexdigest(JSON.generate(stable_sort(value)))
+
+              fact_hash = {
+                id:               SecureRandom.uuid,
+                store:            store,
+                key:              key_map[key_idx.to_i],
+                value:            value,
+                value_hash:       vh,
+                causation:        nil,
+                transaction_time: ts_ms / 1_000.0,
+                valid_time:       valid_time,
+                schema_version:   sv
+              }
+              fact = Fact.from_h(fact_hash) rescue nil
+              facts << fact if fact
+            end
+          end
+
+          facts
+        end
+
+        private
+
+        def write_batch(io)
+          entries = @batch_buf.map do |f|
+            key = f.key.to_s
+            @key_map[key] ||= @key_map.size
+
+            ts_ms   = (f.transaction_time.to_f * 1_000).round
+            delta   = @last_ts_ms ? ts_ms - @last_ts_ms : ts_ms
+            @last_ts_ms = ts_ms
+
+            vals = @fields.map { |fn|
+              v = f.value.key?(fn.to_sym) ? f.value[fn.to_sym] : f.value[fn]
+              v.is_a?(Symbol) ? v.to_s : v
+            }
+            [@key_map[key], delta, vals]
+          end
+
+          new_keys = @key_map.select { |_, idx| idx >= @km_flushed }
+                             .invert
+                             .transform_keys(&:to_s)
+          @km_flushed = @key_map.size
+
+          raw  = MessagePack.pack(stringify({ km: new_keys, e: entries }))
+          body = [@batch_buf.size].pack("N") + Zlib::Deflate.deflate(raw, Zlib::BEST_COMPRESSION)
+          @batch_buf.clear
+          io.write(encode_frame(body))
+        end
+
+        def stringify(v)
+          case v
+          when Symbol then v.to_s
+          when Hash   then v.transform_keys(&:to_s).transform_values { |x| stringify(x) }
+          when Array  then v.map { |x| stringify(x) }
+          else             v
+          end
+        end
+
+        def stable_sort(v)
+          case v
+          when Hash  then v.sort_by { |k, _| k.to_s }.to_h { |k, x| [k.to_s, stable_sort(x)] }
+          when Array then v.map { |x| stable_sort(x) }
+          else            v
+          end
+        end
+      end
+    end
+  end
+end

data/lib/igniter/store/contractable_receipt_sink.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Store
+    # Durable sink for contractable observation/event receipts emitted by
+    # igniter-embed's contractable runner.
+    #
+    # Implements the record_observation / record_event store adapter protocol
+    # so it can be passed directly as the `store:` option to any contractable.
+    #
+    # Idempotency policy:
+    #   record_observation — keyed by observation_id; same id overwrites the
+    #     current fact and creates a causation chain entry. Safe to retry.
+    #   record_event — append-only history; retries produce duplicate entries.
+    #     Callers should deduplicate at the source if needed.
+    class ContractableReceiptSink
+      REQUIRED_OBSERVATION_FIELDS = %i[observation_id receipt_kind].freeze
+      REQUIRED_EVENT_FIELDS = %i[event_id receipt_kind observation_id].freeze
+
+      attr_reader :store, :client, :observations_store, :events_store, :producer
+
+      def initialize(
+        store: nil,
+        client: nil,
+        observations_store: :contractable_observations,
+        events_store: :contractable_events,
+        producer: { type: :embed, name: :contractable_receipt_sink }
+      )
+        raise ArgumentError, "ContractableReceiptSink requires store: or client:" unless store || client
+
+        @store = store
+        @client = client
+        @observations_store = observations_store.to_sym
+        @events_store = events_store.to_sym
+        @producer = producer
+        register_descriptors
+      end
+
+      def record_observation(receipt)
+        validate_receipt!(receipt, REQUIRED_OBSERVATION_FIELDS, :contractable_observation)
+        target.write(
+          store: observations_store,
+          key: receipt[:observation_id].to_s,
+          value: receipt,
+          producer: producer
+        )
+      end
+
+      def record_event(receipt)
+        validate_receipt!(receipt, REQUIRED_EVENT_FIELDS, :contractable_event)
+        target.append(
+          history: events_store,
+          event: receipt,
+          partition_key: :observation_id,
+          producer: producer
+        )
+      end
+
+      def observation(observation_id)
+        normalize_read_result(target.read(store: observations_store, key: observation_id.to_s))
+      end
+
+      def events_for(observation_id)
+        history_partition_values(
+          store: events_store,
+          partition_key: :observation_id,
+          partition_value: observation_id.to_s
+        )
+      end
+
+      def observations(status: nil, limit: nil)
+        all_facts = history_facts(store: observations_store)
+        by_key = {}
+        all_facts.each { |f| by_key[fact_key(f)] = f }
+        results = by_key.values.sort_by { |f| fact_transaction_time(f) }.map { |f| fact_value(f) }
+        results = results.select { |r| r[:status] == status } if status
+        limit ? results.take(limit) : results
+      end
+
+      def error_events(limit: nil)
+        results = history_facts(store: events_store).map { |f| fact_value(f) }.select { |r| r[:severity] == :error }
+        limit ? results.take(limit) : results
+      end
+
+      private
+
+      def target
+        client || store
+      end
+
+      def validate_receipt!(receipt, required_fields, expected_kind)
+        missing = required_fields.select { |f| receipt[f].nil? }
+        raise ArgumentError, "contractable receipt missing required fields: #{missing.join(", ")}" if missing.any?
+
+        actual_kind = receipt[:receipt_kind]
+        return if actual_kind == expected_kind
+
+        raise ArgumentError, "expected receipt_kind #{expected_kind.inspect}, got #{actual_kind.inspect}"
+      end
+
+      def register_descriptors
+        target.register_descriptor(
+          kind: :store,
+          name: observations_store,
+          key: :observation_id,
+          fields: %i[observation_id name role stage status sampled async started_at finished_at duration_ms redaction],
+          producer: { system: :igniter_embed }
+        )
+        target.register_descriptor(
+          kind: :history,
+          name: events_store,
+          key: :event_id,
+          partition_key: :observation_id,
+          fields: %i[event_id observation_id event severity summary occurred_at]
+        )
+      end
+
+      def normalize_read_result(result)
+        return result.value if result.respond_to?(:value) && result.respond_to?(:found?)
+
+        if result.is_a?(Hash) && result.key?(:value)
+          result[:value]
+        else
+          result
+        end
+      end
+
+      def history_partition_values(store:, partition_key:, partition_value:)
+        if target.respond_to?(:history_partition)
+          return target.history_partition(
+            store: store,
+            partition_key: partition_key,
+            partition_value: partition_value
+          ).map(&:value)
+        end
+
+        history_facts(store: store)
+          .map { |f| fact_value(f) }
+          .select { |value| value[partition_key] == partition_value }
+      end
+
+      def history_facts(store:)
+        return target.history(store: store) if target.respond_to?(:history)
+
+        replay_result = target.replay(store: store)
+        return replay_result.facts if replay_result.respond_to?(:facts)
+
+        if replay_result.is_a?(Hash) && replay_result.key?(:facts)
+          replay_result[:facts]
+        else
+          Array(replay_result)
+        end
+      end
+
+      def fact_key(fact)
+        fact.is_a?(Hash) ? fact[:key] : fact.key
+      end
+
+      def fact_value(fact)
+        fact.is_a?(Hash) ? fact[:value] : fact.value
+      end
+
+      def fact_transaction_time(fact)
+        if fact.is_a?(Hash)
+          fact[:transaction_time] || fact[:timestamp] || 0
+        else
+          fact.transaction_time
+        end
+      end
+    end
+  end
+end

data/lib/igniter/store/fact.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "securerandom"
+
+module Igniter
+  module Store
+    unless defined?(NATIVE) && NATIVE
+      # Pure-Ruby Fact Struct — skipped when the Rust native extension is loaded.
+      # The native extension provides its own Fact class with :build and reader methods.
+      Fact = Struct.new(
+        :id,
+        :store,
+        :key,
+        :value,
+        :value_hash,
+        :causation,
+        :transaction_time,
+        :valid_time,
+        :schema_version,
+        :producer,
+        :derivation,
+        keyword_init: true
+      ) do
+        # Canonical build entry point.
+        # valid_time: domain time (writer-supplied, nullable Float).
+        # term: backward-compat alias for valid_time — accepted but deprecated.
+        def self.build(store:, key:, value:, causation: nil, valid_time: nil, term: nil,
+                       schema_version: 1, producer: nil, derivation: nil)
+          vt = valid_time.nil? ? (term ? term.to_f : nil) : valid_time.to_f
+          serialized = JSON.generate(stable_sort(value))
+          new(
+            id:               SecureRandom.uuid,
+            store:            store,
+            key:              key,
+            value:            deep_freeze(value),
+            value_hash:       Digest::SHA256.hexdigest(serialized),
+            causation:        causation,
+            transaction_time: Process.clock_gettime(Process::CLOCK_REALTIME),
+            valid_time:       vt,
+            schema_version:   schema_version,
+            producer:         producer ? deep_freeze(producer) : nil,
+            derivation:       derivation ? deep_freeze(derivation) : nil
+          ).freeze
+        end
+
+        private_class_method def self.stable_sort(value)
+          case value
+          when Hash
+            value.sort_by { |key, _entry| key.to_s }.to_h do |key, entry|
+              [key.to_s, stable_sort(entry)]
+            end
+          when Array
+            value.map { |entry| stable_sort(entry) }
+          else
+            value
+          end
+        end
+
+        private_class_method def self.deep_freeze(value)
+          case value
+          when Hash
+            value.transform_values { |entry| deep_freeze(entry) }.freeze
+          when Array
+            value.map { |entry| deep_freeze(entry) }.freeze
+          else
+            value.frozen? ? value : value.dup.freeze
+          end
+        end
+
+        # Backward-compat: callers that read fact.timestamp still work.
+        alias_method :timestamp, :transaction_time
+        # Backward-compat: callers that read fact.term still work.
+        alias_method :term, :valid_time
+      end
+    end
+
+    # Reopen Fact (Ruby Struct or native class) and add from_h + normalizations.
+    class Fact
+      if defined?(Igniter::Store::NATIVE) && Igniter::Store::NATIVE
+        # Native extension stores `store` as a Rust String; normalize to Symbol
+        # to match the Ruby Struct fallback behaviour.
+        alias_method :_native_store_str, :store
+        def store = _native_store_str.to_sym
+      end
+
+      # Reconstruct a Fact from a wire-deserialized hash.
+      # Accepts both old field names (timestamp, term) and new names
+      # (transaction_time, valid_time) for smooth transition.
+      #
+      # In native mode: Fact.new is unavailable (no Ruby allocator), so we call
+      # Fact.build which recomputes id and transaction_time. This is a known
+      # Phase 2 gap for time-travel fidelity over the network.
+      def self.from_h(h)
+        h = h.transform_keys(&:to_sym)
+        h[:store]    = h.fetch(:store).to_sym
+        # Accept both old (timestamp) and new (transaction_time) key names.
+        h[:transaction_time] = (h[:transaction_time] || h[:timestamp])&.to_f || 0.0
+        h[:valid_time]       = (h[:valid_time] || h[:term])&.to_f
+
+        if defined?(Igniter::Store::NATIVE) && Igniter::Store::NATIVE
+          build(
+            store:        h[:store],
+            key:          h[:key],
+            value:        h[:value],
+            causation:    h[:causation],
+            valid_time:   h[:valid_time],
+            schema_version: h.fetch(:schema_version, 1),
+            producer:     h[:producer],
+            derivation:   h[:derivation]
+          )
+        else
+          new(**h.slice(:id, :store, :key, :value, :value_hash, :causation,
+                        :transaction_time, :valid_time, :schema_version,
+                        :producer, :derivation)).freeze
+        end
+      end
+    end
+  end
+end

data/lib/igniter/store/fact_log.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "monitor"
+
+module Igniter
+  module Store
+    unless defined?(NATIVE) && NATIVE
+      # Pure-Ruby FactLog — skipped when the Rust native extension is loaded.
+      class FactLog
+        include MonitorMixin
+
+        def initialize
+          super()
+          @log = []
+          @by_id = {}
+          @by_key = Hash.new { |hash, key| hash[key] = [] }
+        end
+
+        def append(fact)
+          synchronize do
+            @log << fact
+            @by_id[fact.id] = fact
+            @by_key[[fact.store, fact.key]] << fact
+          end
+          fact
+        end
+
+        def replay(fact)
+          synchronize do
+            @log << fact
+            @by_id[fact.id] = fact
+            @by_key[[fact.store, fact.key]] << fact
+          end
+        end
+
+        def latest_for(store:, key:, as_of: nil)
+          facts = synchronize { @by_key[[store, key]].dup }
+          facts = facts.select { |fact| fact.transaction_time <= as_of } if as_of
+          facts.last
+        end
+
+        def facts_for(store:, key: nil, since: nil, as_of: nil)
+          synchronize do
+            facts = key ? @by_key[[store, key]].dup : @log.select { |fact| fact.store == store }
+            facts = facts.select { |fact| fact.transaction_time >= since } if since
+            facts = facts.select { |fact| fact.transaction_time <= as_of } if as_of
+            facts
+          end
+        end
+
+        def query_scope(store:, filters:, as_of: nil)
+          synchronize do
+            seen = {}
+            @by_key.each do |(s, k), facts|
+              next unless s == store
+              candidates = as_of ? facts.select { |f| f.transaction_time <= as_of } : facts
+              latest = candidates.last
+              next unless latest
+              seen[k] = latest if matches_filters?(latest.value, filters)
+            end
+            seen.values
+          end
+        end
+
+        def all_facts
+          synchronize { @log.dup }
+        end
+
+        def size
+          synchronize { @log.size }
+        end
+
+        private
+
+        def matches_filters?(value, filters)
+          return false unless value.is_a?(Hash)
+          filters.all? { |k, v| value[k] == v }
+        end
+      end
+    end
+
+    if defined?(NATIVE) && NATIVE
+      # Patch the Rust-native FactLog to expose all_facts.
+      # The native append is intercepted to track which stores have been written;
+      # all_facts then collects via facts_for(store:) per known store.
+      class FactLog
+        alias_method :_native_append_unwrapped, :append
+
+        def append(fact)
+          @_seen_stores ||= []
+          s = fact.store
+          @_seen_stores << s unless @_seen_stores.include?(s)
+          _native_append_unwrapped(fact)
+        end
+
+        def all_facts
+          @_seen_stores ||= []
+          @_seen_stores.flat_map { |s| facts_for(store: s) }.sort_by(&:transaction_time)
+        end
+      end
+    end
+  end
+end