nats-pure 2.3.0 → 2.5.0

data/lib/nats/io/kv.rb

@@ -1,4 +1,6 @@
-# Copyright 2021 The NATS Authors
+# frozen_string_literal: true
+
+# Copyright 2021-2025 The NATS Authors
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,13 +14,15 @@
 # limitations under the License.
 #
 
-require_relative 'kv/api'
-require_relative 'kv/bucket_status'
-require_relative 'kv/errors'
-require_relative 'kv/manager'
+require_relative "kv/api"
+require_relative "kv/bucket_status"
+require_relative "kv/errors"
+require_relative "kv/manager"
 
 module NATS
   class KeyValue
+    include MonitorMixin
+
     KV_OP = "KV-Operation"
     KV_DEL = "DEL"
     KV_PURGE = "PURGE"
@@ -26,16 +30,35 @@ module NATS
     MSG_ROLLUP_ALL = "all"
     ROLLUP = "Nats-Rollup"
 
-    def initialize(opts={})
+    VALID_BUCKET_RE = /\A[a-zA-Z0-9_-]+$/
+    VALID_KEY_RE = /\A[-\/_=\.a-zA-Z0-9]+$/
+
+    class << self
+      def is_valid_key(key)
+        if key.nil?
+          false
+        elsif key.start_with?(".") || key.end_with?(".")
+          false
+        elsif key !~ VALID_KEY_RE
+          false
+        else
+          true
+        end
+      end
+    end
+
+    def initialize(opts = {})
       @name = opts[:name]
       @stream = opts[:stream]
       @pre = opts[:pre]
       @js = opts[:js]
       @direct = opts[:direct]
+      @validate_keys = opts[:validate_keys]
     end
 
     # get returns the latest value for the key.
-    def get(key, params={})
+    def get(key, params = {})
+      raise InvalidKeyError if @validate_keys && !KeyValue.is_valid_key(key)
       entry = nil
       begin
         entry = _get(key, params)
@@ -46,19 +69,19 @@ module NATS
       entry
     end
 
-    def _get(key, params={})
+    def _get(key, params = {})
       msg = nil
       subject = "#{@pre}#{key}"
 
-      if params[:revision]
-        msg = @js.get_msg(@stream,
-                          seq: params[:revision],
-                          direct: @direct)
+      msg = if params[:revision]
+        @js.get_msg(@stream,
+          seq: params[:revision],
+          direct: @direct)
       else
-        msg = @js.get_msg(@stream,
-                          subject: subject,
-                          seq: params[:revision],
-                          direct: @direct)
+        @js.get_msg(@stream,
+          subject: subject,
+          seq: params[:revision],
+          direct: @direct)
       end
 
       entry = Entry.new(bucket: @name, key: key, value: msg.data, revision: msg.seq)
@@ -70,9 +93,9 @@ module NATS
         )
       end
 
-      if not msg.headers.nil?
+      if !msg.headers.nil?
         op = msg.headers[KV_OP]
-        if op == KV_DEL or op == KV_PURGE
+        if (op == KV_DEL) || (op == KV_PURGE)
           raise KeyDeletedError.new(entry: entry, op: op)
         end
       end
@@ -86,12 +109,16 @@ module NATS
     # put will place the new value for the key into the store
     # and return the revision number.
     def put(key, value)
+      raise InvalidKeyError if @validate_keys && !KeyValue.is_valid_key(key)
+
       ack = @js.publish("#{@pre}#{key}", value)
       ack.seq
     end
 
     # create will add the key/value pair iff it does not exist.
     def create(key, value)
+      raise InvalidKeyError if @validate_keys && !KeyValue.is_valid_key(key)
+
       pa = nil
       begin
         pa = update(key, value, last: 0)
@@ -122,7 +149,9 @@ module NATS
     EXPECTED_LAST_SUBJECT_SEQUENCE = "Nats-Expected-Last-Subject-Sequence"
 
     # update will update the value iff the latest revision matches.
-    def update(key, value, params={})
+    def update(key, value, params = {})
+      raise InvalidKeyError if @validate_keys && !KeyValue.is_valid_key(key)
+
       hdrs = {}
       last = (params[:last] ||= 0)
       hdrs[EXPECTED_LAST_SUBJECT_SEQUENCE] = last.to_s
@@ -141,7 +170,9 @@ module NATS
     end
 
     # delete will place a delete marker and remove all previous revisions.
-    def delete(key, params={})
+    def delete(key, params = {})
+      raise InvalidKeyError if @validate_keys && !KeyValue.is_valid_key(key)
+
       hdrs = {}
       hdrs[KV_OP] = KV_DEL
       last = (params[:last] ||= 0)
@@ -155,6 +186,8 @@ module NATS
 
     # purge will remove the key and all revisions.
     def purge(key)
+      raise InvalidKeyError if @validate_keys && !KeyValue.is_valid_key(key)
+
       hdrs = {}
       hdrs[KV_OP] = KV_PURGE
       hdrs[ROLLUP] = MSG_ROLLUP_SUBJECT
@@ -168,11 +201,315 @@ module NATS
     end
 
     Entry = Struct.new(:bucket, :key, :value, :revision, :delta, :created, :operation, keyword_init: true) do
-      def initialize(opts={})
+      def initialize(opts = {})
         rem = opts.keys - members
         opts.delete_if { |k| rem.include?(k) }
-        super(opts)
+        super
       end
     end
+
+    # watch will be signaled when any key is updated.
+    def watchall(params = {})
+      watch(">", params)
+    end
+
+    # keys returns the keys from a KeyValue store.
+    # Optionally filters the keys based on the provided filter list.
+    def keys(params = {})
+      params[:ignore_deletes] = true
+      params[:meta_only] = true
+
+      w = watchall(params)
+      got_keys = false
+
+      Enumerator.new do |y|
+        w.each do |entry|
+          break if entry.nil?
+          got_keys = true
+          y << entry.key
+        end
+        w.stop
+        raise NoKeysFoundError unless got_keys
+      end
+    end
+
+    # history retrieves the entries so far for a key.
+    def history(key, params = {})
+      params[:include_history] = true
+      w = watch(key, params)
+      got_keys = false
+
+      Enumerator.new do |y|
+        w.each do |entry|
+          break if entry.nil?
+          got_keys = true
+          y << entry
+        end
+        w.stop
+        raise NoKeysFoundError unless got_keys
+      end
+    end
+
+    STATUS_HDR = "Status"
+    DESC_HDR = "Description"
+    CTRL_STATUS = "100"
+    LAST_CONSUMER_SEQ_HDR = "Nats-Last-Consumer"
+    LAST_STREAM_SEQ_HDR = "Nats-Last-Stream"
+    CONSUMER_STALLED_HDR = "Nats-Consumer-Stalled"
+
+    # watch will be signaled when a key that matches the keys
+    # pattern is updated.
+    # The first update after starting the watch is nil in case
+    # there are no pending updates.
+    def watch(keys, params = {})
+      params[:meta_only] ||= false
+      params[:include_history] ||= false
+      params[:ignore_deletes] ||= false
+      params[:idle_heartbeat] ||= 5 # seconds
+      params[:inactive_threshold] ||= 5 * 60 # 5 minutes
+      subject = "#{@pre}#{keys}"
+      init_setup = new_cond
+      init_setup_done = false
+      nc = @js.nc
+      watcher = KeyWatcher.new(@js)
+
+      deliver_policy = if !(params[:include_history])
+        "last_per_subject"
+      end
+
+      ordered = {
+        # basic ordered consumer.
+        flow_control: true,
+        ack_policy: "none",
+        max_deliver: 1,
+        ack_wait: 22 * 3600,
+        idle_heartbeat: params[:idle_heartbeat],
+        num_replicas: 1,
+        mem_storage: true,
+        manual_ack: true,
+        # watch related options.
+        deliver_policy: deliver_policy,
+        headers_only: params[:meta_only],
+        inactive_threshold: params[:inactive_threshold]
+      }
+
+      # watch_updates callback.
+      sub = @js.subscribe(subject, config: ordered) do |msg|
+        synchronize do
+          if !init_setup_done
+            init_setup.wait(@js.opts[:timeout])
+          end
+        end
+
+        # Control Message like Heartbeats and Flow Control
+        status = msg.header[STATUS_HDR] unless msg.header.nil?
+        if !status.nil? && status == CTRL_STATUS
+          desc = msg.header[DESC_HDR]
+          if desc.start_with?("Idle")
+            # A watcher is active if it continues to receive Idle Heartbeat messages.
+            #
+            # Status: 100
+            # Description: Idle Heartbeat
+            # Nats-Last-Consumer: 185
+            # Nats-Last-Stream: 185
+            #
+            watcher.synchronize { watcher._active = true }
+          elsif desc.start_with?("FlowControl")
+            # HMSG _INBOX.q6Y3JAFxOnNJi4QdwQnFtg 2 $JS.FC.KV_TEST.t00CunIG.GT4W 36 36
+            # NATS/1.0 100 FlowControl Request
+            nc.publish(msg.reply)
+          end
+          # Skip processing the control message
+          next
+        end
+
+        # Track sequences
+        meta = msg.metadata
+        watcher.synchronize { watcher._active = true }
+        # Track the sequences
+        #
+        # $JS.ACK.KV_TEST.CKRGrWpf.1.10.10.1739859923871837000.0
+        #
+        tokens = msg.reply.split(".")
+        sseq = tokens[5]
+        dseq = tokens[6]
+        watcher.synchronize do
+          watcher._dseq = dseq.to_i + 1
+          watcher._sseq = sseq.to_i
+        end
+
+        # Keys() handling
+        op = nil
+        if msg.header && msg.header[KV_OP]
+          op = msg.header[KV_OP]
+          if params[:ignore_deletes]
+            if (op == KV_PURGE) || (op == KV_DEL)
+              if (meta.num_pending == 0) && !watcher._init_done
+                # Push this to unblock enumerators.
+                watcher._updates.push(nil)
+                watcher._init_done = true
+              end
+              next
+            end
+          end
+        end
+
+        # Convert the msg into an Entry.
+        key = msg.subject[@pre.size...msg.subject.size]
+        entry = Entry.new(
+          bucket: @name,
+          key: key,
+          value: msg.data,
+          revision: meta.sequence.stream,
+          delta: meta.num_pending,
+          created: meta.timestamp,
+          operation: op
+        )
+        watcher._updates.push(entry)
+
+        # When there are no more updates send an empty marker
+        # to signal that it is done, this will unblock iterators.
+        if (meta.num_pending == 0) && !watcher._init_done
+          watcher._updates.push(nil)
+          watcher._init_done = true
+        end
+      end # end of callback
+      watcher._sub = sub
+
+      # Snapshot the deliver subject for the consumer.
+      deliver_subject = sub.subject
+
+      # Check from consumer info what is the number of messages
+      # awaiting to be consumed to send the initial signal marker.
+      stream_name = nil
+      begin
+        cinfo = sub.consumer_info
+        stream_name = cinfo.stream_name
+
+        synchronize do
+          init_setup_done = true
+          # If no delivered and/or pending messages, then signal
+          # that this is the start.
+          # The consumer subscription will start receiving messages
+          # so need to check those that have already made it.
+          received = sub.delivered
+          init_setup.signal
+
+          # When there are no more updates send an empty marker
+          # to signal that it is done, this will unblock iterators.
+          if (cinfo.num_pending == 0) && (received == 0)
+            watcher._updates.push(nil)
+            watcher._init_done = true
+          end
+        end
+      rescue => err
+        # cancel init
+        sub.unsubscribe
+        raise err
+      end
+
+      # Need to handle reconnect if missing too many heartbeats.
+      hb_interval = params[:idle_heartbeat] * 2
+      watcher._hb_task = Concurrent::TimerTask.new(execution_interval: hb_interval) do |task|
+        task.shutdown if nc.closed?
+        next unless nc.connected?
+
+        # Wait for all idle heartbeats to be received, one of them would have
+        # toggled the state of the consumer back to being active.
+        active = watcher.synchronize {
+          current = watcher._active
+          # A heartbeat or another incoming message needs to toggle back.
+          watcher._active = false
+          current
+        }
+        if !active
+          ccreq = ordered.dup
+          ccreq[:deliver_policy] = "by_start_sequence"
+          ccreq[:opt_start_seq] = watcher._sseq
+          ccreq[:deliver_subject] = deliver_subject
+          ccreq[:idle_heartbeat] = ordered[:idle_heartbeat]
+          ccreq[:inactive_threshold] = ordered[:inactive_threshold]
+
+          should_recreate = false
+          begin
+            # Check if the original is still present, if it is then do not recreate.
+            begin
+              sub.consumer_info
+            rescue ::NATS::JetStream::Error::ConsumerNotFound => e
+              e.stream ||= sub.jsi.stream
+              e.consumer ||= sub.jsi.consumer
+              @js.nc.send(:err_cb_call, @js.nc, e, sub)
+              should_recreate = true
+            end
+            next unless should_recreate
+
+            # Recreate consumer that went away after a restart.
+            cinfo = @js.add_consumer(stream_name, ccreq)
+            sub.jsi.consumer = cinfo.name
+            watcher.synchronize { watcher._dseq = 1 }
+          rescue => e
+            # Dispatch to the error NATS client error callback.
+            @js.nc.send(:err_cb_call, @js.nc, e, sub)
+          end
+        end
+      rescue => e
+        # WRN: Unexpected error
+        @js.nc.send(:err_cb_call, @js.nc, e, sub)
+      end
+      watcher._hb_task.execute
+
+      watcher
+    end
+  end
+
+  class KeyWatcher
+    include MonitorMixin
+    include Enumerable
+    attr_accessor :received, :pending, :_sub, :_updates, :_init_done, :_watcher_cond
+    attr_accessor :_sseq, :_dseq, :_active, :_hb_task
+
+    def initialize(js)
+      super() # required to initialize monitor
+      @js = js
+      @_sub = nil
+      @_updates = SizedQueue.new(256)
+      @_init_done = false
+      @pending = nil
+      # Ordered consumer related
+      @_dseq = 1
+      @_sseq = 0
+      @_cmeta = nil
+      @_fcr = 0
+      @_fciseq = 0
+      @_active = true
+      @_hb_task = nil
+    end
+
+    def stop
+      @_hb_task.shutdown
+      @_sub.unsubscribe
+    end
+
+    def updates(params = {})
+      params[:timeout] ||= 5
+      result = nil
+      MonotonicTime.with_nats_timeout(params[:timeout]) do
+        result = @_updates.pop(timeout: params[:timeout])
+      end
+
+      result
+    end
+
+    # Implements Enumerable.
+    def each
+      loop do
+        result = @_updates.pop
+        yield result
+      end
+    end
+
+    def take(n)
+      super.take(n).reject { |entry| entry.nil? }
+    end
   end
 end

data/lib/nats/io/msg.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Copyright 2016-2021 The NATS Authors
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -10,36 +12,34 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-require_relative 'jetstream'
 
 module NATS
   class Msg
     attr_accessor :subject, :reply, :data, :header
+    attr_accessor :nc, :sub
 
-    # Enhance it with ack related methods from JetStream to ack msgs.
-    include JetStream::Msg::AckMethods
-
-    def initialize(opts={})
+    def initialize(opts = {})
       @subject = opts[:subject]
-      @reply   = opts[:reply]
-      @data    = opts[:data]
-      @header  = opts[:header]
-      @nc      = opts[:nc]
-      @sub     = opts[:sub]
-      @ackd    = false
-      @meta    = nil
+      @reply = opts[:reply]
+      @data = opts[:data]
+      @header = opts[:header]
+      @nc = opts[:nc]
+      @sub = opts[:sub]
+
+      # JS related
+      @ackd = false
+      @meta = nil
     end
 
-    def respond(data='')
+    def respond(data = "")
       return unless @nc
-      if self.header
-        dmsg = self.dup
-        dmsg.subject = self.reply
+      if header
+        dmsg = dup
+        dmsg.subject = reply
         dmsg.data = data
         @nc.publish_msg(dmsg)
       else
-        @nc.publish(self.reply, data)
+        @nc.publish(reply, data)
       end
     end
 
@@ -50,7 +50,7 @@ module NATS
 
     def inspect
       hdr = ", header=#{@header}" if @header
-      dot = '...' if @data.length > 10
+      dot = "..." if @data.length > 10
       dat = "#{data.slice(0, 10)}#{dot}"
       "#<NATS::Msg(subject: \"#{@subject}\", reply: \"#{@reply}\", data: #{dat.inspect}#{hdr})>"
     end

data/lib/nats/io/parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Copyright 2016-2018 The NATS Authors
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,27 +16,26 @@
 
 module NATS
   module Protocol
-
-    MSG      = /\AMSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\r\n/i
-    HMSG     = /\AHMSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?([\d]+)\s+(\d+)\r\n/i
-    OK       = /\A\+OK\s*\r\n/i
-    ERR      = /\A-ERR\s+('.+')?\r\n/i
-    PING     = /\APING\s*\r\n/i
-    PONG     = /\APONG\s*\r\n/i
-    INFO     = /\AINFO\s+([^\r\n]+)\r\n/i
-    UNKNOWN  = /\A(.*)\r\n/
+    MSG = /\AMSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\r\n/i
+    HMSG = /\AHMSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?([\d]+)\s+(\d+)\r\n/i
+    OK = /\A\+OK\s*\r\n/i
+    ERR = /\A-ERR\s+('.+')?\r\n/i
+    PING = /\APING\s*\r\n/i
+    PONG = /\APONG\s*\r\n/i
+    INFO = /\AINFO\s+([^\r\n]+)\r\n/i
+    UNKNOWN = /\A(.*)\r\n/
 
     AWAITING_CONTROL_LINE = 1
-    AWAITING_MSG_PAYLOAD  = 2
+    AWAITING_MSG_PAYLOAD = 2
 
-    CR_LF = ("\r\n".freeze)
-    CR_LF_SIZE = (CR_LF.bytesize)
+    CR_LF = "\r\n"
+    CR_LF_SIZE = CR_LF.bytesize
 
-    PING_REQUEST  = ("PING#{CR_LF}".freeze)
-    PONG_RESPONSE = ("PONG#{CR_LF}".freeze)
+    PING_REQUEST = "PING#{CR_LF}".freeze
+    PONG_RESPONSE = "PONG#{CR_LF}".freeze
 
-    SUB_OP = ('SUB'.freeze)
-    EMPTY_MSG = (''.freeze)
+    SUB_OP = "SUB"
+    EMPTY_MSG = ""
 
     class Parser
       def initialize(nc)
@@ -55,7 +56,7 @@ module NATS
 
       def parse(data)
         @buf = @buf ? @buf << data : data
-        while (@buf)
+        while @buf
           case @parse_state
           when AWAITING_CONTROL_LINE
             case @buf
@@ -91,23 +92,22 @@ module NATS
               # If we are here we do not have a complete line yet that we understand.
               return
             end
-            @buf = nil if (@buf && @buf.empty?)
+            @buf = nil if @buf && @buf.empty?
 
           when AWAITING_MSG_PAYLOAD
-            return unless (@needed && @buf.bytesize >= (@needed + CR_LF_SIZE))
+            return unless @needed && @buf.bytesize >= (@needed + CR_LF_SIZE)
             if @header_needed
               hbuf = @buf.slice(0, @header_needed)
-              payload = @buf.slice(@header_needed, (@needed-@header_needed))
+              payload = @buf.slice(@header_needed, (@needed - @header_needed))
               @nc.send(:process_msg, @sub, @sid, @reply, payload, hbuf)
-              @buf = @buf.slice((@needed + CR_LF_SIZE), @buf.bytesize)
             else
               @nc.send(:process_msg, @sub, @sid, @reply, @buf.slice(0, @needed), nil)
-              @buf = @buf.slice((@needed + CR_LF_SIZE), @buf.bytesize)
             end
+            @buf = @buf.slice((@needed + CR_LF_SIZE), @buf.bytesize)
 
             @sub = @sid = @reply = @needed = @header_needed = nil
             @parse_state = AWAITING_CONTROL_LINE
-            @buf = nil if (@buf && @buf.empty?)
+            @buf = nil if @buf && @buf.empty?
           end
         end
       end

data/lib/nats/io/rails.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "rails"
 
 module NATS