loams 0.1.0 → 0.1.2

data/lib/loams/client.rb

@@ -1,25 +1,71 @@
 # frozen_string_literal: true
 
+require "json"
 require "loams/descriptors"
 require "loams/transport"
 require "loams/invoker"
 require "loams/pagination"
+require "loams/consistency"
+require "loams/app"
 
 module Loams
   class Client
-    attr_reader :invoker, :endpoint
+    attr_reader :invoker, :endpoint, :namespace, :session
 
+    # The app modules (SDKP §2), hand-written over the descriptor set's
+    # bindings. They take the place of the descriptor-derived module of the
+    # same name, whose raw calls stay reachable through `client.module(name)`.
+    APP_MODULES = {
+      "namespaces" => App::Namespaces,
+      "collections" => App::Collections,
+      "documents" => App::Documents,
+      "search" => App::SearchModule,
+      "vector" => App::VectorModule,
+      "graph" => App::Graph,
+      "sql" => App::Sql,
+      "streams" => App::Streams,
+    }.freeze
+
+    # A client from the environment (R1): `LOAMS_ENDPOINT`, then
+    # `LOAMS_API_KEY` or `LOAMS_TOKEN`, then `LOAMS_NAMESPACE`. Keyword
+    # arguments win over the environment.
+    def self.from_env(env = ENV, **options)
+      endpoint = env["LOAMS_ENDPOINT"]
+      if endpoint.nil? || endpoint.empty?
+        raise InvalidArgumentError.new("LOAMS_ENDPOINT is not set", code: :invalid_argument)
+      end
+
+      key = env["LOAMS_API_KEY"]
+      token = env["LOAMS_TOKEN"]
+      defaults = {}
+      defaults[:api_key] = key unless key.nil? || key.empty?
+      defaults[:token] = token if defaults[:api_key].nil? && !(token.nil? || token.empty?)
+      ns = env["LOAMS_NAMESPACE"]
+      defaults[:namespace] = ns unless ns.nil? || ns.empty?
+      new(endpoint, **defaults, **options)
+    end
+
+    # +api_key+ is sent as `Authorization: Bearer` (R1), as are +token+ and
+    # +token_source+. +namespace+ is the default namespace (R11). +timeout+
+    # is the unary default in seconds, nil for none (R12).
+    # +session_consistency+ turns on the session store (R4).
     def initialize(
       endpoint = "http://127.0.0.1:4400",
       protocol: :connect,
       codec: :proto,
+      api_key: nil,
       token: nil,
       token_source: nil,
+      namespace: nil,
+      timeout: CallInvoker::DEFAULT_TIMEOUT,
+      session_consistency: false,
       transport: nil,
       max_retries: 3
     )
       @endpoint = endpoint
-      tokens = token_source || token
+      @namespace = namespace&.to_s
+      @session = session_consistency ? SessionStore.new : nil
+      tokens = token_source || api_key || token
       retries = RetryPolicy.new(max_retries: max_retries)
       tp = transport || HttpTransport.new(endpoint)
 
@@ -29,6 +75,7 @@ module Loams
         codec: codec,
         token_source: tokens,
         retry_policy: retries,
+        timeout: timeout,
       )
 
       # Build module clients
@@ -40,12 +87,39 @@ module Loams
         method_name = Descriptors.snake(mod.name)
         define_singleton_method(method_name) { client }
       end
+
+      APP_MODULES.each do |name, klass|
+        app = klass.new(self)
+        define_singleton_method(name) { app }
+      end
     end
 
     def module(name)
       @modules[name.to_s]
     end
 
+    # Any facade call by its facade-index name (`documents.write`,
+    # `instance.get_instance`), over its request's proto3 JSON or message.
+    # `headers:`, `timeout:` and `idempotency_key:` are the call's options.
+    def call(name, request = {}, **opts)
+      mod, call = name.split(".", 2)
+      if App::CALLS.key?([mod, call])
+        (@raw ||= App::Raw.new(self)).call(name, request, **opts)
+      else
+        client = @modules[mod] or raise ArgumentError, "no module #{mod}"
+        binding = client.module_binding.calls.find { |b| Descriptors.snake(b.facade_name) == call }
+        raise ArgumentError, "no call #{name}" if binding.nil?
+
+        message = request.is_a?(Hash) ? binding.request.msgclass.decode_json(JSON.generate(request)) : request
+        client.public_send(call, message, opts)
+      end
+    end
+
+    # A handle on one collection (SDKP §2.8).
+    def collection(name, namespace: nil)
+      CollectionHandle.new(self, name, namespace: namespace)
+    end
+
     class ModuleClient
       attr_reader :module_binding, :invoker
 

data/lib/loams/consistency.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+require "loams/errors"
+
+module Loams
+  # A consistency token (runtime contract R4): `v1:s<stream>/p<partition>@<offset>,…`,
+  # the encoding `crates/loams-collection/src/token.rs` defines.
+  #
+  # A write answers with one and a read accepts one as `at_least`, so a caller
+  # that just wrote can read its own write. Two tokens merge by keeping the
+  # largest offset per `(stream, partition)`, which is what lets a session fold
+  # every answer it sees into one token.
+  class ConsistencyToken
+    PREFIX = "v1:"
+    U32_MAX = 0xFFFF_FFFF
+    U64_MAX = 0xFFFF_FFFF_FFFF_FFFF
+
+    # The `(stream, partition, offset)` triples, in the order the token
+    # spells them. `merge` (and `normalized`) sort them, one per
+    # `(stream, partition)`.
+    attr_reader :items
+
+    # Parses +text+, raising `InvalidArgumentError` (client-side, no network
+    # call) when it is not a `v1:` token.
+    def self.parse(text)
+      raise invalid(text) unless text.is_a?(String) && text.start_with?(PREFIX)
+
+      body = text[PREFIX.length..]
+      return new([]) if body.empty?
+
+      items = body.split(",", -1).map do |item|
+        parse_item(item) || raise(invalid(text))
+      end
+      new(items)
+    end
+
+    # Whether +text+ parses.
+    def self.valid?(text)
+      parse(text)
+      true
+    rescue LoamsError
+      false
+    end
+
+    def initialize(items = [])
+      @items = items.map { |i| i.dup.freeze }.freeze
+      freeze
+    end
+
+    # The same token sorted by stream then partition, one item per pair (the
+    # largest offset).
+    def normalized
+      self.class.new(normalize(@items))
+    end
+
+    # The token's text, `v1:` followed by the triples.
+    def to_s
+      PREFIX + @items.map { |s, p, o| "s#{s}/p#{p}@#{o}" }.join(",")
+    end
+
+    # A new token holding the larger offset of the two for every
+    # `(stream, partition)`.
+    def merge(other)
+      other = self.class.parse(other) if other.is_a?(String)
+      self.class.new(normalize(@items + other.items))
+    end
+
+    # The offset this token holds for one partition, or nil.
+    def offset(stream, partition)
+      @items.select { |s, p, _| s == stream && p == partition }.map(&:last).max
+    end
+
+    def empty?
+      @items.empty?
+    end
+
+    def ==(other)
+      other.is_a?(ConsistencyToken) && other.items == @items
+    end
+    alias eql? ==
+
+    def hash
+      @items.hash
+    end
+
+    def inspect
+      "#<Loams::ConsistencyToken #{self}>"
+    end
+
+    def self.parse_item(item)
+      return nil unless item.start_with?("s")
+
+      stream, rest = item[1..].split("/p", 2)
+      return nil if rest.nil?
+
+      partition, offset = rest.split("@", 2)
+      return nil if offset.nil?
+
+      s = canonical(stream, U64_MAX)
+      p = canonical(partition, U32_MAX)
+      o = canonical(offset, U64_MAX)
+      return nil if s.nil? || p.nil? || o.nil?
+
+      [s, p, o]
+    end
+    private_class_method :parse_item
+
+    # A decimal with no sign, no leading zero (except `0`) and in range.
+    def self.canonical(text, max)
+      return nil unless text.match?(/\A(0|[1-9][0-9]*)\z/)
+
+      n = Integer(text, 10)
+      n <= max ? n : nil
+    end
+    private_class_method :canonical
+
+    def self.invalid(text)
+      InvalidArgumentError.new("invalid consistency token #{text.inspect}", code: :invalid_argument, reason: :invalid_consistency_token)
+    end
+    private_class_method :invalid
+
+    private
+
+    def normalize(items)
+      items.sort.each_with_object([]) do |item, out|
+        if out.last && out.last[0] == item[0] && out.last[1] == item[1]
+          out[-1] = item
+        else
+          out << item
+        end
+      end.freeze
+    end
+  end
+
+  # The session store (R4, `session_consistency: true`): every token an
+  # answer carries is merged in, and every read that leaves `consistency`
+  # unset is sent with `at_least` set to the session's token.
+  #
+  # One token per proto package: a graph's tokens name the graph's own
+  # stream and a graph read refuses a token of another stream, so the
+  # collection and graph sessions are kept apart.
+  class SessionStore
+    # The response fields a token is read from.
+    TOKEN_FIELDS = %w[token read_token consistency_token].freeze
+
+    def initialize
+      @tokens = {}
+      @lock = Mutex.new
+    end
+
+    # The session token for +scope+ (a proto package), or nil.
+    def token(scope = "loams.collection.v1")
+      @lock.synchronize { @tokens[scope] }
+    end
+
+    # Merges +text+ into the session for +scope+. A token that does not
+    # parse is ignored: the session never holds a token it cannot merge.
+    def observe(scope, text)
+      return if text.nil? || text.empty?
+      return unless ConsistencyToken.valid?(text)
+
+      parsed = ConsistencyToken.parse(text)
+      @lock.synchronize do
+        current = @tokens[scope]
+        @tokens[scope] = current ? current.merge(parsed) : parsed
+      end
+    end
+
+    # Reads every token field off +message+ and merges it in.
+    def observe_message(scope, message)
+      return if message.nil? || !message.respond_to?(:class) || !message.class.respond_to?(:descriptor)
+
+      TOKEN_FIELDS.each do |name|
+        field = message.class.descriptor.lookup(name)
+        next if field.nil? || field.type != :string || field.label == :repeated
+
+        observe(scope, message[name])
+      end
+    end
+
+    # Sets `consistency.at_least` on +request+ when it has a `consistency`
+    # field the caller left unset and the session holds a token.
+    def apply(scope, request)
+      return request unless request.respond_to?(:class) && request.class.respond_to?(:descriptor)
+
+      field = request.class.descriptor.lookup("consistency")
+      return request if field.nil? || field.type != :message
+      return request unless request[field.name].nil?
+
+      current = token(scope)
+      return request if current.nil? || current.empty?
+
+      sub = field.subtype.msgclass.new
+      return request if sub.class.descriptor.lookup("at_least").nil?
+
+      sub["at_least"] = current.to_s
+      request[field.name] = sub
+      request
+    end
+  end
+end

data/lib/loams/descriptors.rb

@@ -88,6 +88,12 @@ module Loams
       collect_messages(file.message_type, file.package, "")
     end
 
+    # Every message's class, built up front. `google-protobuf` builds a
+    # class on first `msgclass`, and iterating a repeated message field whose
+    # element class was never built raises `TypeError` (a `SearchResponse`'s
+    # `hits`), so no class is left to be built lazily.
+    MESSAGES.each_value { |descriptor| descriptor&.msgclass }
+
     # Every service in the set, in the order the files declare them.
     SERVICES = SET.file.flat_map do |file|
       file.service.to_a.map do |raw_svc|

data/lib/loams/errors.rb

@@ -54,6 +54,7 @@ module Loams
     :idempotency_key_reused,
     :workspace_not_found,
     :workspace_fenced,
+    :batch_partially_applied,
     :workspace_read_only,
     :workspace_name_taken,
     :object_missing,
@@ -61,17 +62,46 @@ module Loams
     :env_unsealed,
     :env_access_denied,
     :env_foreign_reference,
+    :memory_store_not_found,
+    :memory_space_not_found,
+    :memory_space_limit,
+    :memory_managed_object,
+    :memory_block_version_mismatch,
+    :memory_block_too_large,
+    :memory_core_budget_exceeded,
+    :memory_content_too_large,
+    :memory_secret_detected,
+    :memory_pii_refused,
+    :memory_model_not_configured,
+    :memory_model_not_local,
+    :memory_token_budget_exhausted,
+    :memory_extraction_failed,
+    :memory_too_many_spaces,
+    :memory_cross_space_write,
+    :memory_not_yet_available,
+    :code_repo_not_indexed,
+    :code_index_building,
+    :code_change_too_large,
+    :code_language_unsupported,
+    :code_scip_invalid,
     :live_bad_cursor,
     :live_worker_crashed,
     :live_deploy_conflict,
     :live_schema_violation,
+    :filter_conflict,
+    :search_after_needs_tiebreak,
+    :invalid_consistency_token,
+    :namespace_required,
+    :sql_param_count,
     :live_index_not_ready,
     :live_app_disabled,
     :live_app_exists,
     :live_function_error,
     :live_function_timeout,
+    :client_deadline_exceeded,
     :live_function_out_of_memory,
     :live_limit_exceeded,
+    :result_too_large,
     :live_app_mismatch,
     :live_origin_not_allowed,
     :session_rotate,
@@ -159,6 +189,7 @@ module Loams
     idempotency_key_reused: :failed_precondition,
     workspace_not_found: :not_found,
     workspace_fenced: :aborted,
+    batch_partially_applied: :aborted,
     workspace_read_only: :failed_precondition,
     workspace_name_taken: :already_exists,
     object_missing: :failed_precondition,
@@ -166,17 +197,46 @@ module Loams
     env_unsealed: :invalid_argument,
     env_access_denied: :permission_denied,
     env_foreign_reference: :permission_denied,
+    memory_store_not_found: :not_found,
+    memory_space_not_found: :not_found,
+    memory_space_limit: :resource_exhausted,
+    memory_managed_object: :failed_precondition,
+    memory_block_version_mismatch: :aborted,
+    memory_block_too_large: :invalid_argument,
+    memory_core_budget_exceeded: :resource_exhausted,
+    memory_content_too_large: :invalid_argument,
+    memory_secret_detected: :failed_precondition,
+    memory_pii_refused: :failed_precondition,
+    memory_model_not_configured: :failed_precondition,
+    memory_model_not_local: :failed_precondition,
+    memory_token_budget_exhausted: :resource_exhausted,
+    memory_extraction_failed: :failed_precondition,
+    memory_too_many_spaces: :invalid_argument,
+    memory_cross_space_write: :invalid_argument,
+    memory_not_yet_available: :unimplemented,
+    code_repo_not_indexed: :failed_precondition,
+    code_index_building: :unavailable,
+    code_change_too_large: :invalid_argument,
+    code_language_unsupported: :failed_precondition,
+    code_scip_invalid: :invalid_argument,
     live_bad_cursor: :invalid_argument,
     live_worker_crashed: :internal,
     live_deploy_conflict: :failed_precondition,
     live_schema_violation: :invalid_argument,
+    filter_conflict: :invalid_argument,
+    search_after_needs_tiebreak: :invalid_argument,
+    invalid_consistency_token: :invalid_argument,
+    namespace_required: :invalid_argument,
+    sql_param_count: :invalid_argument,
     live_index_not_ready: :failed_precondition,
     live_app_disabled: :failed_precondition,
     live_app_exists: :failed_precondition,
     live_function_error: :unknown,
     live_function_timeout: :deadline_exceeded,
+    client_deadline_exceeded: :deadline_exceeded,
     live_function_out_of_memory: :resource_exhausted,
     live_limit_exceeded: :resource_exhausted,
+    result_too_large: :resource_exhausted,
     live_app_mismatch: :permission_denied,
     live_origin_not_allowed: :permission_denied,
     session_rotate: :unavailable,

data/lib/loams/invoker.rb

@@ -7,18 +7,25 @@ require "loams/idempotency"
 require "loams/retry"
 require "loams/token_source"
 require "loams/streams"
+require "net/http"
+require "timeout"
 
 module Loams
   class CallInvoker
-    attr_reader :transport, :protocol, :codec, :token_source, :retry_policy
+    # The client default for a unary call, in seconds (R12). Streams have none.
+    DEFAULT_TIMEOUT = 30
+
+    attr_reader :transport, :protocol, :codec, :token_source, :retry_policy, :timeout
 
     def initialize(
       transport:,
       protocol: :connect,
       codec: :proto,
       token_source: nil,
-      retry_policy: nil
+      retry_policy: nil,
+      timeout: DEFAULT_TIMEOUT
     )
+      @timeout = timeout
       @transport = transport
       @protocol = protocol
       @codec = codec
@@ -33,9 +40,15 @@ module Loams
 
       attempt = 0
       refreshed_token = false
+      # R12: one budget for the call, retries included. `timeout: nil` on the
+      # call or the client means no deadline.
+      budget = options.key?(:timeout) ? options[:timeout] : @timeout
+      deadline = budget && (monotonic + budget.to_f)
 
       loop do
         attempt += 1
+        remaining = deadline && (deadline - monotonic)
+        raise deadline_error(binding.rpc) if remaining && remaining <= 0
 
         # Build headers
         ct = ContentTypes.content_type_for(@protocol, @codec, streaming: false)
@@ -49,6 +62,15 @@ module Loams
           headers["idempotency-key"] = key
         end
 
+        if remaining
+          ms = [(remaining * 1000).ceil, 1].max
+          if @protocol == :grpc_web
+            headers["grpc-timeout"] = "#{ms}m"
+          else
+            headers["connect-timeout-ms"] = ms.to_s
+          end
+        end
+
         if options[:headers]
           options[:headers].each { |k, v| headers[k.to_s.downcase] = v.to_s }
         end
@@ -65,10 +87,14 @@ module Loams
           headers: headers,
           body: body,
           rpc: binding.rpc,
+          timeout: remaining,
         )
 
         res = begin
           @transport.call_unary(req)
+        rescue Net::OpenTimeout, Net::ReadTimeout, Timeout::Error
+          # The budget is spent, so this is never retried.
+          raise deadline_error(binding.rpc)
         rescue StandardError => e
           raise ErrorParser.build_error(:unknown, e.message, nil, nil, binding.rpc)
         end
@@ -192,6 +218,19 @@ module Loams
 
     private
 
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def deadline_error(rpc)
+      DeadlineExceededError.new(
+        "the call ran out of time on the client",
+        code: :deadline_exceeded,
+        reason: :client_deadline_exceeded,
+        rpc: rpc,
+      )
+    end
+
     def current_token(force_refresh: false)
       @token_source&.token(force_refresh: force_refresh)
     end

data/lib/loams/subscribe.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Loams
+  # R14: `streams.subscribe`, a loop over unary `Fetch` (SDKP-2), so it works
+  # on Connect-unary and through any proxy.
+  #
+  # The next offset is the last record's offset + 1. An empty fetch backs off
+  # 100 ms, doubling up to 1 s, and a non-empty one resets the backoff.
+  # `offset` is where the next fetch starts, so a caller can resume from it;
+  # `cancel` stops the loop at the next fetch boundary.
+  #
+  #   sub = client.streams.subscribe("events", 0, from_offset: 1)
+  #   sub.each { |record| … ; sub.cancel if done }
+  class Subscription
+    include Enumerable
+
+    INITIAL_BACKOFF = 0.1
+    MAX_BACKOFF = 1.0
+
+    attr_reader :offset
+
+    # +fetch+ is called with an offset and answers the records at and after
+    # it, each responding to `offset` (or a Hash with `"offset"`).
+    def initialize(from_offset: 0, sleeper: nil, &fetch)
+      raise ArgumentError, "a subscription needs a fetch block" unless fetch
+
+      @offset = Integer(from_offset)
+      @fetch = fetch
+      @sleeper = sleeper || ->(seconds) { sleep(seconds) }
+      @cancelled = false
+    end
+
+    def cancel
+      @cancelled = true
+    end
+
+    def cancelled?
+      @cancelled
+    end
+
+    def each
+      return to_enum(:each) unless block_given?
+
+      backoff = INITIAL_BACKOFF
+      until @cancelled
+        records = Array(@fetch.call(@offset))
+        if records.empty?
+          @sleeper.call(backoff)
+          backoff = [backoff * 2, MAX_BACKOFF].min
+          next
+        end
+        backoff = INITIAL_BACKOFF
+        records.each do |record|
+          break if @cancelled
+
+          @offset = Integer(record_offset(record)) + 1
+          yield record
+        end
+      end
+      self
+    end
+
+    private
+
+    def record_offset(record)
+      if record.respond_to?(:offset)
+        record.offset
+      elsif record.is_a?(Hash)
+        record["offset"] || record[:offset]
+      else
+        raise ArgumentError, "a fetched record has no offset"
+      end
+    end
+  end
+end

data/lib/loams/transport.rb

@@ -4,7 +4,7 @@ require "net/http"
 require "uri"
 
 module Loams
-  Request = Struct.new(:method, :path, :headers, :body, :rpc, keyword_init: true)
+  Request = Struct.new(:method, :path, :headers, :body, :rpc, :timeout, keyword_init: true)
   Response = Struct.new(:status, :headers, :body, :stream, keyword_init: true)
 
   class HttpTransport
@@ -22,8 +22,10 @@ module Loams
       url = URI.join(@endpoint + "/", request.path.sub(%r{^/+}, ""))
       http = Net::HTTP.new(url.host, url.port)
       http.use_ssl = (url.scheme == "https")
-      http.read_timeout = 30
-      http.open_timeout = 5
+      # R12: the call's remaining budget, or no limit when it has none.
+      http.read_timeout = request.timeout
+      http.open_timeout = request.timeout ? [request.timeout, 5].min : 5
+      http.write_timeout = request.timeout if request.timeout
 
       req = Net::HTTPGenericRequest.new(
         request.method || "POST",