igniter-ledger 0.5.2

data/lib/igniter/store/file_backend.rb

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+require "json"
+require "zlib"
+require "set"
+require "fileutils"
+require_relative "wire_protocol"
+
+module Igniter
+  module Store
+    unless defined?(NATIVE) && NATIVE
+      # Pure-Ruby FileBackend — skipped when the Rust native extension is loaded.
+      #
+      # WAL format (v2): length-prefixed frames with CRC32 integrity check.
+      #
+      # Each frame:
+      #   [4 bytes BE uint32: body_len][body_len bytes: JSON][4 bytes BE uint32: CRC32(body)]
+      #
+      # Snapshot format (path + ".snap"):
+      #   [header frame: JSON { type: "snapshot_header", fact_count: N, written_at: T }]
+      #   [fact frame 1] ... [fact frame N]
+      #
+      # On open, if a snapshot file exists: snapshot facts are loaded first, WAL
+      # facts whose IDs are already in the snapshot are skipped. Combined result
+      # is sorted by timestamp. Startup cost is O(snapshot_size + delta_wal_size)
+      # instead of O(total_wal_size).
+      class FileBackend
+        include WireProtocol
+
+        SNAPSHOT_SUFFIX = ".snap"
+
+        def initialize(path)
+          @path = path.to_s
+          @file = File.open(@path, "ab")
+          @file.sync = true
+        end
+
+        def write_fact(fact)
+          body  = JSON.generate(fact.to_h)
+          @file.write(encode_frame(body))
+        end
+
+        # Combines snapshot (if present) + WAL delta into a chronologically
+        # ordered list of facts.  Facts in the snapshot are deduplicated against
+        # the WAL by ID so a checkpoint never causes double-replay.
+        def replay
+          snapshot_facts, seen_ids = load_snapshot
+          wal_facts = read_wal_frames.reject { |f| seen_ids.include?(f.id) }
+          (snapshot_facts + wal_facts).sort_by(&:transaction_time)
+        end
+
+        # Atomically writes all +facts+ to a snapshot file (<wal_path>.snap).
+        # Uses a tmp file + rename so a partial write never corrupts an existing
+        # snapshot.  The WAL file is untouched; the snapshot is a parallel read
+        # artefact only.
+        def write_snapshot(facts)
+          tmp = "#{snapshot_path}.tmp"
+          File.open(tmp, "wb") do |f|
+            header = JSON.generate({
+              type:       "snapshot_header",
+              fact_count: facts.size,
+              written_at: Process.clock_gettime(Process::CLOCK_REALTIME)
+            })
+            f.write(encode_frame(header))
+            facts.each { |fact| f.write(encode_frame(JSON.generate(fact.to_h))) }
+          end
+          FileUtils.mv(tmp, snapshot_path)
+        end
+
+        # Pruning-safe barrier: atomically replace the snapshot with +facts+ AND
+        # truncate the WAL so that dropped facts cannot resurface on reopen.
+        #
+        # Normal checkpoint (#write_snapshot) is non-destructive — it leaves the
+        # WAL intact, which means any fact not present in the snapshot will still
+        # be loaded from the WAL on next open.  For physical purge that is wrong:
+        # the dropped fact ids would not be in the new snapshot, so the WAL would
+        # replay them back into existence.
+        #
+        # This method:
+        #   1. Writes the new snapshot atomically (tmp → rename).
+        #   2. Closes the current WAL file handle.
+        #   3. Truncates the WAL to 0 bytes (new open in write mode).
+        #   4. Reopens for future appends.
+        #
+        # After a successful call, close/reopen will load only the snapshot facts.
+        def replace_with_snapshot!(facts)
+          write_snapshot(facts)
+          @file.close
+          File.open(@path, "wb") {}   # truncate WAL
+          @file = File.open(@path, "ab")
+          @file.sync = true
+        end
+
+        def snapshot_path
+          @path + SNAPSHOT_SUFFIX
+        end
+
+        def close
+          @file.close
+        end
+
+        private
+
+        # Parses a raw JSON body into a frozen Fact.  Returns nil on parse error.
+        def decode_fact(body)
+          payload = JSON.parse(body, symbolize_names: true)
+          Fact.from_h(payload)
+        rescue JSON::ParserError
+          nil
+        end
+
+        # --- WAL reading ---
+
+        def read_wal_frames
+          return [] unless File.exist?(@path)
+          facts = []
+          File.open(@path, "rb") do |f|
+            loop do
+              body = read_frame(f)
+              break unless body
+              fact = decode_fact(body)
+              facts << fact if fact
+            end
+          end
+          facts
+        end
+
+        # --- Snapshot reading ---
+
+        # Returns [Array<Fact>, Set<id>] from the snapshot file, or [[], Set[]]
+        # if no snapshot exists or the snapshot is corrupt.
+        def load_snapshot
+          return [[], Set.new] unless File.exist?(snapshot_path)
+
+          facts = []
+          File.open(snapshot_path, "rb") do |f|
+            header_body = read_frame(f)
+            return [[], Set.new] unless header_body
+
+            header = JSON.parse(header_body, symbolize_names: true)
+            return [[], Set.new] unless header[:type] == "snapshot_header"
+
+            loop do
+              body = read_frame(f)
+              break unless body
+              fact = decode_fact(body)
+              facts << fact if fact
+            end
+          end
+
+          [facts, Set.new(facts.map(&:id))]
+        rescue StandardError
+          [[], Set.new]
+        end
+      end
+    end
+
+    if defined?(NATIVE) && NATIVE
+      # Patch native FileBackend to add snapshot support.
+      # The native class exposes write_fact, replay (WAL-only), and close.
+      # We add: write_snapshot, snapshot_path, SNAPSHOT_SUFFIX, and a
+      # snapshot-aware replay that merges snapshot facts with the WAL delta.
+      #
+      # Deduplication uses the original fact id embedded in the snapshot JSON
+      # (not the id on the reconstructed Fact object, which is regenerated by
+      # Fact.build in native mode).
+
+      module NativeFileBackendSnapshotSupport
+        include WireProtocol
+
+        SNAPSHOT_SUFFIX = ".snap"
+
+        def snapshot_path
+          @_ruby_path + SNAPSHOT_SUFFIX
+        end
+
+        # Pruning-safe barrier for native-backed stores.
+        # Writes the snapshot atomically, then truncates the WAL file so that
+        # dropped facts cannot be replayed on reopen.
+        # The native write handle is still open; after truncation, the next
+        # native write will restart from offset 0 (append mode semantics).
+        def replace_with_snapshot!(facts)
+          write_snapshot(facts)
+          File.open(@_ruby_path, "wb") {}   # truncate WAL
+        end
+
+        def write_snapshot(facts)
+          tmp = "#{snapshot_path}.tmp"
+          File.open(tmp, "wb") do |f|
+            header = JSON.generate({
+              type:       "snapshot_header",
+              fact_count: facts.size,
+              written_at: Process.clock_gettime(Process::CLOCK_REALTIME)
+            })
+            f.write(encode_frame(header))
+            facts.each { |fact| f.write(encode_frame(JSON.generate(fact.to_h))) }
+          end
+          FileUtils.mv(tmp, snapshot_path)
+        end
+
+        # Merges snapshot facts (if any) with WAL facts not already in the snapshot.
+        # Deduplication is by original id read directly from the JSON frame, because
+        # Fact.from_h in native mode regenerates ids via Fact.build.
+        def replay
+          snapshot_facts, seen_ids = load_native_snapshot
+          wal_facts = _native_replay_wal
+          delta = wal_facts.reject { |f| seen_ids.include?(f.id) }
+          snapshot_facts + delta
+        end
+
+        private
+
+        def load_native_snapshot
+          return [[], Set.new] unless File.exist?(snapshot_path)
+
+          facts    = []
+          seen_ids = Set.new
+          File.open(snapshot_path, "rb") do |f|
+            header_body = read_frame(f)
+            return [[], Set.new] unless header_body
+            header = JSON.parse(header_body, symbolize_names: true)
+            return [[], Set.new] unless header[:type] == "snapshot_header"
+
+            loop do
+              body = read_frame(f)
+              break unless body
+              h = JSON.parse(body, symbolize_names: true)
+              seen_ids.add(h[:id]) if h[:id]
+              fact = Fact.from_h(h)
+              facts << fact if fact
+            end
+          end
+          [facts, seen_ids]
+        rescue StandardError
+          [[], Set.new]
+        end
+      end
+
+      class FileBackend
+        include NativeFileBackendSnapshotSupport
+
+        SNAPSHOT_SUFFIX = NativeFileBackendSnapshotSupport::SNAPSHOT_SUFFIX
+
+        # The native extension defines `replay` (WAL-only) as a class-level method,
+        # which shadows the module's snapshot-aware `replay`.  Save the WAL-only
+        # version under an alias, then override `replay` in the class body so the
+        # snapshot-aware path is used on store open.
+        alias_method :_native_replay_wal, :replay
+
+        def replay
+          snapshot_facts, seen_ids = load_native_snapshot
+          wal_facts = _native_replay_wal
+          delta = wal_facts.reject { |f| seen_ids.include?(f.id) }
+          snapshot_facts + delta
+        end
+
+        class << self
+          alias_method :_native_new, :new
+
+          def new(path)
+            obj = _native_new(path)
+            obj.instance_variable_set(:@_ruby_path, path.to_s)
+            obj
+          end
+        end
+      end
+    end
+  end
+end

data/lib/igniter/store/http_adapter.rb

@@ -0,0 +1,413 @@
+# frozen_string_literal: true
+
+require "rack"
+require "json"
+
+module Igniter
+  module Store
+    # HTTP transport adapter for the Igniter Store Open Protocol.
+    #
+    # Exposes Protocol::Interpreter over HTTP via a Rack-compatible app.
+    # The canonical endpoint is POST /v1/dispatch which accepts and returns
+    # a WireEnvelope JSON object.
+    #
+    # Usage:
+    #   adapter = HTTPAdapter.new(interpreter: interpreter, port: 7300)
+    #   adapter.rack_app   # → Rack-compatible, mountable in any server
+    #   adapter.start      # → foreground via Puma (dev dep)
+    #   adapter.start_async / adapter.stop
+    class HTTPAdapter
+      module ResponseHelper
+        private
+
+        def json_response(status, data)
+          body = JSON.generate(data)
+          [status, { "Content-Type" => "application/json", "Content-Length" => body.bytesize.to_s }, [body]]
+        end
+
+        def method_not_allowed
+          json_response(405, { error: "Method not allowed" })
+        end
+      end
+
+      class DispatchHandler
+        include ResponseHelper
+
+        def initialize(interpreter)
+          @interpreter = interpreter
+        end
+
+        def call(env)
+          return method_not_allowed unless env["REQUEST_METHOD"] == "POST"
+
+          body = env["rack.input"].read
+          begin
+            envelope = JSON.parse(body, symbolize_names: true)
+          rescue JSON::ParserError => e
+            return json_response(400, { error: "Invalid JSON: #{e.message}" })
+          end
+
+          json_response(200, @interpreter.wire.dispatch(envelope))
+        end
+      end
+
+      class HealthHandler
+        include ResponseHelper
+
+        def initialize(health_provider: nil)
+          @health_provider = health_provider
+        end
+
+        def call(env)
+          return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
+
+          health = @health_provider ? @health_provider.call : { protocol: :igniter_store, schema_version: 1, status: :ready }
+          json_response(200, health)
+        end
+      end
+
+      class MetadataHandler
+        include ResponseHelper
+
+        def initialize(interpreter)
+          @interpreter = interpreter
+        end
+
+        def call(env)
+          return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
+
+          json_response(200, @interpreter.metadata_snapshot)
+        end
+      end
+
+      # Returns the canonical observability snapshot at GET /v1/status.
+      # When +status_provider+ is given (e.g. StoreServer#observability_snapshot),
+      # it is called to produce the full server+storage shape.
+      # Otherwise falls back to the interpreter's storage-level snapshot.
+      class StatusHandler
+        include ResponseHelper
+
+        def initialize(interpreter:, status_provider: nil)
+          @interpreter     = interpreter
+          @status_provider = status_provider
+        end
+
+        def call(env)
+          return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
+
+          data = @status_provider ? @status_provider.call : @interpreter.observability_snapshot
+          json_response(200, data)
+        end
+      end
+
+      # Readiness probe: 200 when ready to serve traffic, 503 otherwise (draining,
+      # stopped, or initialising).  +ready_provider+ must return truthy/falsy.
+      class ReadyHandler
+        include ResponseHelper
+
+        def initialize(ready_provider: nil)
+          @ready_provider = ready_provider
+        end
+
+        def call(env)
+          return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
+
+          ready = @ready_provider ? @ready_provider.call : true
+          if ready
+            json_response(200, { status: "ready" })
+          else
+            json_response(503, { status: "unavailable" })
+          end
+        end
+      end
+
+      # Returns the metrics sub-hash from the observability snapshot.
+      class MetricsHandler
+        include ResponseHelper
+
+        def initialize(metrics_provider: nil)
+          @metrics_provider = metrics_provider
+        end
+
+        def call(env)
+          return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
+
+          data = @metrics_provider ? @metrics_provider.call : {}
+          json_response(200, data)
+        end
+      end
+
+      # Streaming body for SSE responses.
+      #
+      # Emits retained catch-up events from +replay_events+, then blocks on a
+      # live subscription to the ChangefeedBuffer until #close is called.
+      #
+      # SSE frame format per event:
+      #   id: <sequence>
+      #   event: fact_committed
+      #   data: <ChangeEvent#to_h JSON>
+      #   (blank line)
+      #
+      # #close is safe to call from any thread — it pushes a sentinel to unblock
+      # the live delivery loop so the subscription handle is released cleanly.
+      class SseBody
+        SSE_SENTINEL = :__sse_close
+
+        def initialize(buf, replay_events, sub_stores)
+          @buf           = buf
+          @replay_events = replay_events
+          @sub_stores    = sub_stores
+          @queue         = nil
+        end
+
+        def each
+          @replay_events.each { |e| yield sse_frame(e) }
+
+          @queue  = Queue.new
+          handle  = @buf.subscribe(stores: @sub_stores) { |e| @queue << e }
+
+          begin
+            loop do
+              event = @queue.pop
+              break if event.equal?(SSE_SENTINEL)
+              yield sse_frame(event)
+            end
+          rescue IOError, Errno::EPIPE
+            nil
+          ensure
+            handle&.close
+          end
+        end
+
+        def close
+          @queue&.push(SSE_SENTINEL)
+        end
+
+        private
+
+        def sse_frame(event)
+          "id: #{event.cursor[:sequence]}\nevent: fact_committed\ndata: #{JSON.generate(event.to_h)}\n\n"
+        end
+      end
+
+      # GET /v1/events — Server-Sent Events transport over ChangefeedBuffer.
+      #
+      # Protocol:
+      # 1. Replay retained events (catch-up).
+      # 2. Subscribe for live events.
+      #
+      # Cursor input (both optional):
+      #   Last-Event-ID: N  →  replay after sequence N (browser auto-reconnect)
+      #   ?cursor=N         →  same, for simple clients / tests
+      #
+      # Store filtering (optional):
+      #   ?store=tasks             →  single store
+      #   ?stores=tasks,reminders  →  multiple stores
+      #   (none)                   →  all stores (wildcard)
+      #
+      # Error: when the requested cursor is too old (gap due to ring overflow),
+      # returns 409 JSON instead of starting the stream.
+      class SseEventsHandler
+        include ResponseHelper
+
+        def initialize(changefeed_provider:)
+          @changefeed_provider = changefeed_provider
+        end
+
+        def call(env)
+          return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
+
+          buf = @changefeed_provider&.call
+          return json_response(503, { error: "SSE events endpoint not configured" }) unless buf
+
+          cursor = parse_sse_cursor(env)
+          stores = parse_sse_stores(env)
+
+          replay_result = buf.replay(
+            cursor: cursor,
+            stores: stores.empty? ? nil : stores
+          )
+
+          if replay_result[:status] == :cursor_too_old
+            return json_response(409, {
+              status:        "cursor_too_old",
+              oldest_cursor: replay_result[:oldest_cursor],
+              newest_cursor: replay_result[:newest_cursor],
+              dropped_total: replay_result[:dropped_total]
+            })
+          end
+
+          body = SseBody.new(buf, replay_result[:events], stores)
+          [200, sse_headers, body]
+        end
+
+        private
+
+        def sse_headers
+          {
+            "Content-Type"      => "text/event-stream; charset=utf-8",
+            "Cache-Control"     => "no-cache",
+            "X-Accel-Buffering" => "no"
+          }
+        end
+
+        def parse_sse_cursor(env)
+          last_id = env["HTTP_LAST_EVENT_ID"]
+          if last_id && !last_id.empty?
+            seq = Integer(last_id, 10) rescue nil
+            return { sequence: seq } if seq
+          end
+          query    = Rack::Utils.parse_query(env["QUERY_STRING"] || "")
+          cursor_s = query["cursor"]
+          if cursor_s && !cursor_s.empty?
+            seq = Integer(cursor_s, 10) rescue nil
+            return { sequence: seq } if seq
+          end
+          nil
+        end
+
+        def parse_sse_stores(env)
+          query    = Rack::Utils.parse_query(env["QUERY_STRING"] || "")
+          stores_s = query["stores"] || query["store"] || ""
+          stores_s.split(",").map(&:strip).reject(&:empty?)
+        end
+      end
+
+      # GET /v1/compaction/activity — normalized compaction lifecycle activity.
+      #
+      # Query params (all optional):
+      #   ?store=orders
+      #   ?kind=exact_prune
+      #   ?since=1714000000
+      #   ?limit=50
+      #
+      # Returns same JSON shape as Protocol::Interpreter#compaction_activity.
+      # Non-GET → 405.  Invalid numeric since/limit → 400.
+      class CompactionActivityHandler
+        include ResponseHelper
+
+        def initialize(interpreter)
+          @interpreter = interpreter
+        end
+
+        def call(env)
+          return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
+
+          query = Rack::Utils.parse_query(env["QUERY_STRING"] || "")
+
+          store = query["store"]
+          kind  = query["kind"]
+
+          since_raw = query["since"]
+          if since_raw
+            since = Float(since_raw) rescue nil
+            return json_response(400, { error: "Invalid numeric value for 'since': #{since_raw.inspect}" }) if since.nil?
+          end
+
+          limit_raw = query["limit"]
+          if limit_raw
+            limit = Integer(limit_raw, 10) rescue nil
+            return json_response(400, { error: "Invalid integer value for 'limit': #{limit_raw.inspect}" }) if limit.nil?
+          end
+
+          result = @interpreter.compaction_activity(
+            store: store,
+            kind:  kind,
+            since: since,
+            limit: limit
+          )
+          json_response(200, result)
+        end
+      end
+
+      # Returns recent structured events from the server event ring buffer.
+      class EventsRecentHandler
+        include ResponseHelper
+
+        def initialize(events_provider: nil)
+          @events_provider = events_provider
+        end
+
+        def call(env)
+          return method_not_allowed unless env["REQUEST_METHOD"] == "GET"
+
+          events = @events_provider ? @events_provider.call : []
+          json_response(200, { events: events, count: events.size })
+        end
+      end
+
+      # ── Adapter ──────────────────────────────────────────────────────────────
+
+      def initialize(interpreter:, port: 7300, host: "0.0.0.0",
+                     health_provider: nil, status_provider: nil,
+                     ready_provider: nil, metrics_provider: nil, events_provider: nil,
+                     changefeed_provider: nil)
+        @interpreter         = interpreter
+        @port                = port
+        @host                = host
+        @health_provider     = health_provider
+        @status_provider     = status_provider
+        @ready_provider      = ready_provider
+        @metrics_provider    = metrics_provider
+        @events_provider     = events_provider
+        @changefeed_provider = changefeed_provider
+        @puma                = nil
+        @thread              = nil
+      end
+
+      # Returns a Rack-compatible app mountable in any Rack server.
+      def rack_app
+        interp = @interpreter
+        hp     = @health_provider
+        sp     = @status_provider
+        rp     = @ready_provider
+        mp     = @metrics_provider
+        ep     = @events_provider
+        cp     = @changefeed_provider
+        not_found = ->(env) {
+          body = JSON.generate({ error: "Not found: #{env["REQUEST_METHOD"]} #{env["PATH_INFO"]}" })
+          [404, { "Content-Type" => "application/json", "Content-Length" => body.bytesize.to_s }, [body]]
+        }
+
+        Rack::Builder.new do
+          map "/v1/dispatch"            do run DispatchHandler.new(interp) end
+          map "/v1/health"              do run HealthHandler.new(health_provider: hp) end
+          map "/v1/status"              do run StatusHandler.new(interpreter: interp, status_provider: sp) end
+          map "/v1/ready"               do run ReadyHandler.new(ready_provider: rp) end
+          map "/v1/metrics"             do run MetricsHandler.new(metrics_provider: mp) end
+          # /v1/events/recent must precede /v1/events to avoid prefix shadowing.
+          map "/v1/events/recent"       do run EventsRecentHandler.new(events_provider: ep) end
+          map "/v1/events"              do run SseEventsHandler.new(changefeed_provider: cp) end
+          map "/v1/metadata"            do run MetadataHandler.new(interp) end
+          map "/v1/compaction/activity" do run CompactionActivityHandler.new(interp) end
+          run not_found
+        end
+      end
+
+      # Starts the server in the current thread (blocks). Requires puma.
+      def start
+        require "puma"
+        @puma = Puma::Server.new(rack_app)
+        @puma.add_tcp_listener(@host, @port)
+        @puma.run.join
+      end
+
+      # Starts in a background thread. Returns self.
+      def start_async
+        @thread = Thread.new { start }
+        sleep 0.05
+        self
+      end
+
+      def stop
+        @puma&.stop(true) rescue nil
+        @thread&.join(2) rescue nil
+        self
+      end
+
+      def bind_address
+        "#{@host}:#{@port}"
+      end
+    end
+  end
+end