volcano-sdk 0.34.0 → 0.35.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e228fb6138aee4fa4f6bce0f810386d08c3023ed23b3b98ccaf733ba9f1921a6
-  data.tar.gz: df06c47b64ac87c6d44a7a110d7355ea41fe25354aef328cfdd8b246a1375307
+  metadata.gz: a17106ab631dea9f34c8efbe196bd1636ceb1dd80f2751f3c93570f925176418
+  data.tar.gz: 0d8853a55cadc7f86e34b7c6bf39e751b0474d4f878f9732ed013f6b5715794f
 SHA512:
-  metadata.gz: 1fd53ec98301db0ea75e0e947558664be0fd04a10982279d7eaf71c90f15493d7205c086cd7c11d7038d25d1c68e953fe26253f58fb4cc44df08a72b81968636
-  data.tar.gz: 77a02c19da00001b7103e300dc4666f2ac2f1fa339bcc28dd87782d5a4965729b4bd1d67c9486da5a6ed14c4b24b5a93efdaf1feb2010a55a44af5edeadadd51
+  metadata.gz: a2d0f334b4dd8b5482e26b9146e66ad44c97af567e0db6a12124f0a715033bd8e0c9307cbd75bab5b1ee1c6866f47212b999f1a6c237297b749b6803912331c8
+  data.tar.gz: fdfa815814d401cd36bccfb6bfc1179e9222d0792cc993529d3cd16246942cf5c5d1afaf25c84fb7ea8f642551cb494b394b7884dcba64fe4a8c72946de2ce2e

data/README.md

@@ -522,10 +522,11 @@ owner_client.durable.stop(project_id, "charge-order", handle.id)
 `start` takes a durable function's name or its id and returns a handle, never a
 result: an execution can run for up to 366 days, so its result is read back with `get`.
 It uses the same credential `invoke` does — the active user session, then a
-configured service key, then the anonymous key — and is the only durable
-operation an application credential may perform. An `execution_name` makes the
-start idempotent: starting again under the same name returns the execution that
-already exists rather than beginning a second one.
+configured service key, then the anonymous key — and is, with `messages`, the
+only durable operation an application credential may perform. An
+`execution_name` makes the start idempotent: starting again under the same name
+returns the execution that already exists rather than beginning a second one.
+Only the start that creates the execution returns its `client_token`.
 
 `get`, `list`, and `stop` are owner-scoped and require the project owner's
 platform user token, because an execution is addressed by its id alone.
@@ -601,6 +602,54 @@ expired or cancelled approval, raises `Volcano::Error::ConflictError` (HTTP
 `Volcano::Error::NotFoundError`. A `comment` is optional, up to 2,000
 characters.
 
+### Message a durable execution
+
+A durable function written in JavaScript or Python can wait for a named
+message, and publish messages of its own. Ruby sends them, lists them, and
+follows what the execution publishes:
+
+```ruby
+require "async"
+
+handle = client.durable.start("onboarding", { account_id: "account-7" })
+messages = client.durable.messages
+
+messages.send_message("onboarding", handle.id, "documents-uploaded", { count: 3 },
+                      idempotency_key: "upload-42")
+
+page = messages.list("onboarding", handle.id, direction: "out", after: 0)
+puts page.messages.map(&:name), page.has_more
+
+Async do
+  subscription = messages.subscribe("onboarding", handle.id, client_token: handle.client_token) do |message|
+    puts [message.seq, message.name, message.data]
+  end
+  # ...
+  subscription.unsubscribe
+ensure
+  client.realtime.disconnect
+end.wait
+```
+
+A sent message waits until the execution waits for its name. Repeating a send
+with the same `idempotency_key` returns the stored message; sending to an
+execution that has ended raises `Volcano::Error::ConflictError` with `code`
+`execution_not_running`. `subscribe` delivers each published message once, in
+`seq` order, one block call at a time: those published before the call first,
+then live ones, and after a reconnect or a lost live message the ones it
+missed. It needs a signed-in session and an Async reactor; a client with only a
+client token or the service key raises `Volcano::Error::AuthenticationError`.
+Signing out, signing in again, or setting another session ends the
+subscription; a token refresh does not. So does a reconnect the server refuses
+to restore it on, as when the execution is no longer visible to the session.
+
+The service key reaches every execution. A signed-in user reaches the executions
+they started; the anon key, or another user, passes the execution's
+`client_token:`, which only the creating start returns. Pass `project_id:` to
+`send_message`, `list`, or `subscribe` to use the owner's route with a platform
+user token or a project access token; `subscribe` reads back over it. See the
+[functions guide](https://github.com/Kong/volcano-sdk-ruby/blob/main/docs/functions.md#message-a-durable-execution).
+
 ### Read project logs
 
 ```ruby

data/lib/volcano/client.rb

@@ -118,7 +118,6 @@ module Volcano
     def initialize_facades(socket_factory, reconnect_delay)
       @auth = Auth.new(self, @transport, api_url: @api_url)
       @functions = Functions.new(self, @transport, api_url: @api_url)
-      @durable = Durable.new(self, @transport)
       @logs = Logs.new(self, @transport)
       @storage = Storage.new(self, @transport, api_url: @api_url, anon_key: @anon_key)
       @locks = Locks.new(self, @transport)
@@ -129,6 +128,7 @@ module Volcano
         socket_factory: socket_factory,
         reconnect_delay: reconnect_delay
       )
+      @durable = Durable.new(self, @transport, @realtime)
     end
   end
 end

data/lib/volcano/durable.rb

@@ -12,21 +12,26 @@ module Volcano
     # Lists, reads, and decides the approvals durable workflows request.
     attr_reader :approvals
 
-    def initialize(client, transport)
+    # Sends to, lists, and follows the messages of durable executions.
+    attr_reader :messages
+
+    def initialize(client, transport, realtime)
       @client = client
       @transport = transport
       @approvals = DurableApprovals.new(client, transport)
+      @messages = DurableMessages.new(client, transport, realtime)
     end
 
     # Starts an execution and returns its handle. A durable function is never
     # invoked synchronously: it can outlive any request a caller could hold
     # open, so its result is read back with #get.
     #
-    # Starting is the only durable operation an application credential may
-    # perform, and it takes the token an invoke takes. Passing an
+    # Starting takes the token an invoke takes, and is, with #messages, the
+    # only durable operation an application credential may perform. Passing an
     # +execution_name+ makes the start idempotent: starting again under the
     # same name returns the execution that already exists rather than
-    # beginning a second one.
+    # beginning a second one. Only the start that created the execution
+    # returns its +client_token+.
     def start(function_name, payload = {}, execution_name: nil)
       # A durable function's id is accepted here as well as its name, so the
       # DNS-safe name pattern an invoke checks would reject half the input.

data/lib/volcano/durable_approval_models.rb

@@ -12,7 +12,7 @@ module Volcano
   ].freeze
   private_constant :DURABLE_APPROVAL_ATTRIBUTES, :DURABLE_APPROVAL_STATS_ATTRIBUTES
 
-  # Checks keywords for the approval records with too many fields for an
+  # Checks keywords for the durable records with too many fields for an
   # explicit keyword list, and owns immutable copies of their values.
   module DurableApprovalAttributes
     def self.capture(attributes, names, optional:)

data/lib/volcano/durable_message_arguments.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Volcano
+  # Checks durable message arguments before the generated client does, for the
+  # reason DurableArguments gives.
+  module DurableMessageArguments
+    MESSAGE_NAME = /\A[ -~]{1,237}\z/
+    HEADER_VALUE = /\A[ -~]+\z/
+    MAX_IDEMPOTENCY_KEY_LENGTH = 255
+    MAX_CLIENT_TOKEN_LENGTH = 128
+    MAX_LIST_LIMIT = 500
+
+    private
+
+    def known_options(options, known)
+      unknown = options.keys - known
+      raise ArgumentError, "unknown keywords: #{unknown.join(', ')}" unless unknown.empty?
+
+      options
+    end
+
+    def message_name(value)
+      return value.dup.freeze if value.is_a?(String) && MESSAGE_NAME.match?(value)
+
+      raise ArgumentError, 'name must be 1 to 237 printable ASCII characters'
+    end
+
+    # Sent as a header, which carries printable ASCII only.
+    def bounded_text(value, field, limit)
+      return if value.nil?
+      return value.dup.freeze if value.is_a?(String) && HEADER_VALUE.match?(value) && value.length <= limit
+
+      raise ArgumentError, "#{field} must be 1 to #{limit} printable ASCII characters"
+    end
+
+    def after_argument(value)
+      return value if value.nil? || (value.is_a?(Integer) && !value.negative?)
+
+      raise ArgumentError, 'after must be a non-negative Integer'
+    end
+
+    def limit_argument(value)
+      return value if value.nil? || (value.is_a?(Integer) && value.between?(1, MAX_LIST_LIMIT))
+
+      raise ArgumentError, "limit must be an Integer between 1 and #{MAX_LIST_LIMIT}"
+    end
+  end
+  private_constant :DurableMessageArguments
+end

data/lib/volcano/durable_message_backlog.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Volcano
+  # The live messages a durable message stream holds until the ones before
+  # them are delivered. Past LIMIT the oldest is dropped. Each one held after a
+  # dropped one names a predecessor at or past it, so it waits for a read that
+  # lists the dropped one.
+  class DurableMessageBacklog
+    LIMIT = 128
+    private_constant :LIMIT
+
+    def initialize
+      @held = []
+    end
+
+    def empty? = @held.empty?
+
+    def clear = @held.clear
+
+    def hold(held)
+      @held << held
+      @held.delete(oldest) if @held.size > LIMIT
+      nil
+    end
+
+    # Takes the oldest held message if the one its broadcast names as the one
+    # before it is delivered.
+    def take_following(last_seq)
+      held = oldest
+      @held.delete(held) if held && held.prev_seq <= last_seq
+    end
+
+    private
+
+    def oldest = @held.min_by { |candidate| candidate.message.seq }
+  end
+  private_constant :DurableMessageBacklog
+end

data/lib/volcano/durable_message_catch_ups.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Volcano
+  # Runs a durable message stream's own work one job at a time: the catch-ups
+  # it starts itself, the release of the live messages it holds, and the wait
+  # before retrying a catch-up that failed, half a second doubling to 30
+  # seconds. It runs in a task of its own rather than the realtime
+  # connection's, which a reconnect stops midway.
+  class DurableMessageCatchUps
+    FIRST_RETRY_SECONDS = 0.5
+    MAX_RETRY_SECONDS = 30.0
+    private_constant :FIRST_RETRY_SECONDS, :MAX_RETRY_SECONDS
+
+    # +read+ runs a catch-up that was asked for, +release+ delivers the held
+    # live messages, +due+ asks for a catch-up once a retry's wait is over, and
+    # +on_failure+ hears about a job that failed.
+    def initialize(read, release, due, on_failure)
+      @read = read
+      @release = release
+      @due = due
+      @on_failure = on_failure
+      @backoff = FIRST_RETRY_SECONDS
+      @retry_delay = @task = @wake = nil
+      @pending = @releasing = @closed = false
+    end
+
+    # A transient child of the caller's task, so call it from the task that
+    # subscribes. Stopping that task, as a disconnect stops a realtime
+    # callback's, does not stop a transient child, and one that ends hands it
+    # to its parent.
+    def start
+      @task ||= Async::Task.current.async(transient: true) { work }
+      nil
+    end
+
+    def request
+      @pending = true
+      wake.signal
+      nil
+    end
+
+    def release
+      @releasing = true
+      wake.signal
+      nil
+    end
+
+    # Whether a catch-up asked for has yet to start reading.
+    def pending? = @pending
+
+    def retry_later
+      return if @closed
+
+      @retry_delay = @backoff
+      @backoff = [@backoff * 2, MAX_RETRY_SECONDS].min
+      wake.signal
+      nil
+    end
+
+    def cancel_retry = @retry_delay = nil
+
+    def caught_up = @backoff = FIRST_RETRY_SECONDS
+
+    # Not stopped: the catch-up it is running may be what closed it.
+    def close
+      @closed = true
+      wake.signal
+      nil
+    end
+
+    private
+
+    def work
+      until @closed
+        job = next_job
+        job ? run(job) : await_request
+      end
+    end
+
+    # A catch-up goes first: it releases the held messages itself.
+    def next_job
+      if @pending
+        @pending = false
+        @read
+      elsif @releasing
+        @releasing = false
+        @release
+      end
+    end
+
+    def run(job)
+      job.call
+    rescue StandardError => e
+      @on_failure.call(e)
+    end
+
+    def await_request
+      delay = @retry_delay
+      return wake.wait unless delay
+
+      Async::Task.current.with_timeout(delay) { wake.wait }
+    rescue Async::TimeoutError
+      @due.call
+    end
+
+    def wake
+      require 'async/notification'
+      @wake ||= Async::Notification.new
+    end
+  end
+  private_constant :DurableMessageCatchUps
+end

data/lib/volcano/durable_message_models.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Volcano
+  DURABLE_MESSAGE_ATTRIBUTES = %i[
+    id execution_id direction seq name data sender status sent_at delivered_at
+  ].freeze
+  private_constant :DURABLE_MESSAGE_ATTRIBUTES
+
+  # Who sent a durable message. `type` is the credential it was sent with:
+  # `service_key`, `user`, `anon`, `project` for the owner or a project access
+  # token, or `execution` for a message the execution published. `user_id` is
+  # nil unless `type` is `user`.
+  DurableMessageSender = Data.define(:type, :user_id)
+
+  # Reopens the generated record for checked initialization.
+  class DurableMessageSender
+    # @dynamic type, user_id, members, with, to_h, deconstruct, deconstruct_keys
+    # @dynamic self.[], self.members
+    def initialize(type:, user_id: nil)
+      type = type.dup.freeze
+      user_id = user_id&.dup&.freeze
+      super
+    end
+  end
+
+  # A message sent to a durable execution (`direction` `in`), or one it
+  # published (`out`).
+  #
+  # `seq` orders an execution's messages in both directions. It increases with
+  # every message but can skip numbers; list the ones after it with `after:`.
+  # `delivered_at` is nil until the execution receives the message, and for
+  # `out` messages. `data` is frozen all the way down.
+  DurableMessage = Data.define(*DURABLE_MESSAGE_ATTRIBUTES)
+
+  # Reopens the generated record for checked initialization.
+  class DurableMessage
+    # @dynamic id, execution_id, direction, seq, name, data, sender, status, sent_at
+    # @dynamic delivered_at, members, with, to_h, deconstruct, deconstruct_keys
+    # @dynamic self.[], self.members
+    def initialize(**attributes)
+      values = DurableApprovalAttributes.capture(
+        attributes, DURABLE_MESSAGE_ATTRIBUTES, optional: %i[data delivered_at]
+      )
+      attributes = values
+      super
+    end
+  end
+
+  # Messages after a `seq`, oldest first. `has_more` says whether more follow;
+  # read them with `after:` set to the last message's `seq`.
+  DurableMessagePage = Data.define(:messages, :has_more)
+
+  # Reopens the generated record for checked initialization.
+  class DurableMessagePage
+    # @dynamic messages, has_more, members, with, to_h, deconstruct, deconstruct_keys
+    # @dynamic self.[], self.members
+    def initialize(messages:, has_more:)
+      messages = messages.to_a.dup.freeze
+      super
+    end
+  end
+end

data/lib/volcano/durable_message_responses.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module Volcano
+  # Builds durable messages and message pages from transport payloads.
+  module DurableMessageResponses
+    INCOMPLETE_MESSAGE = 'Expected a complete durable message'
+    INCOMPLETE_PAGE = 'Expected a complete durable message page'
+    DIRECTIONS = %w[in out].freeze
+    private_constant :INCOMPLETE_MESSAGE, :INCOMPLETE_PAGE, :DIRECTIONS
+
+    private
+
+    def durable_message(payload)
+      values = message_object(payload, INCOMPLETE_MESSAGE)
+      id, execution_id, name = message_labels(values)
+      direction, status = message_states(values)
+      DurableMessage.new(
+        id: id, execution_id: execution_id, direction: direction, seq: message_seq(values['seq']), name: name,
+        data: message_data(values['data']), sender: message_sender(values['sender']), status: status,
+        sent_at: message_time(values['sent_at']), delivered_at: optional_message_time(values['delivered_at'])
+      )
+    end
+
+    def message_labels(values)
+      [message_text(values['id']), message_text(values['execution_id']), message_text(values['name'])]
+    end
+
+    # The stream filters on direction, so it has to be one it knows. A status
+    # added later reads as written, as an execution's does.
+    def message_states(values)
+      [message_choice(values['direction'], DIRECTIONS), message_text(values['status'])]
+    end
+
+    # The seq a broadcast says the execution published before it, or 0, naming
+    # no gap, when it has none.
+    def broadcast_prev_seq(payload)
+      prev_seq = message_object(payload, INCOMPLETE_MESSAGE)['prev_seq']
+      prev_seq.is_a?(Integer) ? prev_seq : 0
+    end
+
+    def durable_message_page(payload)
+      values = message_object(payload, INCOMPLETE_PAGE)
+      data = values['data']
+      raise TypeError, INCOMPLETE_PAGE unless data.is_a?(Array)
+
+      messages = data.map { |entry| durable_message(entry) }
+      DurableMessagePage.new(messages: messages, has_more: page_flag(values['has_more']))
+    end
+
+    def page_flag(value)
+      return value if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+
+      raise TypeError, INCOMPLETE_PAGE
+    end
+
+    def message_object(payload, message)
+      raise TypeError, message unless payload.is_a?(Hash)
+
+      payload
+    end
+
+    def message_text(value)
+      raise TypeError, INCOMPLETE_MESSAGE unless value.is_a?(String) && !value.empty?
+
+      value
+    end
+
+    def message_choice(value, choices)
+      choices.find { |choice| choice == value } || raise(TypeError, INCOMPLETE_MESSAGE)
+    end
+
+    def message_seq(value)
+      raise TypeError, INCOMPLETE_MESSAGE unless value.is_a?(Integer) && value.positive?
+
+      value
+    end
+
+    def message_data(value)
+      Transport.json_value(value)
+    rescue TypeError
+      raise TypeError, INCOMPLETE_MESSAGE, cause: nil
+    end
+
+    def message_sender(payload)
+      values = message_object(payload, INCOMPLETE_MESSAGE)
+      user_id = values['user_id']
+      raise TypeError, INCOMPLETE_MESSAGE unless user_id.nil? || user_id.is_a?(String)
+
+      DurableMessageSender.new(type: message_text(values['type']), user_id: user_id)
+    end
+
+    def message_time(value)
+      raise TypeError, INCOMPLETE_MESSAGE unless value.is_a?(String)
+
+      Time.iso8601(value)
+    rescue ArgumentError
+      raise TypeError, INCOMPLETE_MESSAGE, cause: nil
+    end
+
+    def optional_message_time(value)
+      value.nil? ? nil : message_time(value)
+    end
+  end
+  private_constant :DurableMessageResponses
+end

data/lib/volcano/durable_message_session.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Volcano
+  # Binds a subscription to the session signed in when it opened, as Postgres
+  # delivery does. Signing out, signing in again, or setting another session
+  # ends it; refreshing its token does not. Once ended, the stream reads and
+  # delivers nothing more, rather than another user's messages, and the
+  # subscription leaves the execution's channel.
+  #
+  # It follows the channel for the stream: a reconnect restores the channel
+  # only while the session is the one that subscribed.
+  class DurableMessageSession
+    CHANGED = "The session that subscribed to the execution's messages has changed; " \
+              'the subscription has ended'
+    private_constant :CHANGED
+
+    def initialize(realtime)
+      @realtime = realtime
+      _, @lineage, = realtime.__send__(:capture_session_binding)
+    end
+
+    # A stream that delivers to +handler+ and reads with +list_after+ while
+    # the session is unchanged, and raises Error::SessionChangedError after.
+    def stream(handler, list_after, on_failure)
+      @stream = DurableMessageStream.new(bind(handler), bind_read(list_after), on_failure)
+    end
+
+    # Ends +subscription+ once the session changes. Called before the
+    # subscription subscribes, so before anything checks the session.
+    def end_with(subscription)
+      @subscription = subscription
+    end
+
+    # Asked before a reconnect restores the channel.
+    def restoring = require_session
+
+    # Told when the server refuses to restore the channel for good.
+    def refused = @subscription.unsubscribe
+
+    def resubscribed = @stream.catch_up_in_background
+    def dropped = @stream.catch_up_in_background
+    def close = @stream.close
+
+    private
+
+    def bind(callable)
+      lambda do |argument|
+        require_session
+        callable.call(argument)
+      end
+    end
+
+    # A page read while the session changed is not delivered, even an empty
+    # one, which delivers nothing that would check it.
+    def bind_read(list_after)
+      lambda do |after|
+        require_session
+        list_after.call(after).tap { require_session }
+      end
+    end
+
+    def require_session
+      _, lineage, session = @realtime.__send__(:capture_session_binding)
+      return if session && lineage == @lineage
+
+      @stream.close
+      @subscription.unsubscribe
+      raise Error::SessionChangedError, CHANGED
+    end
+  end
+  private_constant :DurableMessageSession
+end