couchbase 3.8.2 → 3.9.0

data/ext/couchbase/core/query_stream_component.cxx

@@ -0,0 +1,243 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *   Copyright 2026. Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include "query_stream_component.hxx"
+
+#include "free_form_http_request.hxx"
+#include "http_component.hxx"
+#include "impl/bootstrap_error.hxx"
+#include "logger/logger.hxx"
+#include "operations/document_query.hxx"
+#include "platform/uuid.h"
+#include "service_type.hxx"
+#include "utils/json.hxx"
+
+#include <couchbase/error_codes.hxx>
+
+#include <asio/io_context.hpp>
+#include <spdlog/fmt/bundled/core.h>
+#include <tao/json/value.hpp>
+
+#include <chrono>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <system_error>
+#include <utility>
+
+namespace couchbase::core
+{
+namespace
+{
+auto
+to_error_code(std::error_code err) -> std::error_code
+{
+  return err;
+}
+
+auto
+build_streaming_query_body(const operations::query_request& request,
+                           const std::string& client_context_id,
+                           std::chrono::milliseconds timeout) -> std::string
+{
+  tao::json::value body{};
+
+  // The remaining options are identical to the buffered query path; share one encoder so the two
+  // stay byte-for-byte consistent. The use_replica capability gate is enforced earlier in
+  // cluster::query_stream. This runs first so the control fields set below always win over any
+  // same-named `raw` entry a caller may have supplied.
+  operations::encode_query_options(body, request);
+
+  auto timeout_for_service = timeout;
+  if (timeout_for_service > std::chrono::milliseconds(5'000)) {
+    // Tell the query engine its deadline is 500ms tighter than ours, so we always observe the
+    // server-side timeout response before our own deadline fires.
+    timeout_for_service -= std::chrono::milliseconds(500);
+  }
+  body["client_context_id"] = client_context_id;
+  body["statement"] = request.statement;
+  body["timeout"] = fmt::format("{}ms", timeout_for_service.count());
+
+  return utils::json::generate(body);
+}
+} // namespace
+
+class query_stream_component_impl : public std::enable_shared_from_this<query_stream_component_impl>
+{
+public:
+  query_stream_component_impl(asio::io_context& io,
+                              http_component http,
+                              std::chrono::milliseconds default_timeout,
+                              row_streamer_options streaming_options)
+    : io_{ io }
+    , http_{ std::move(http) }
+    , default_timeout_{ default_timeout }
+    , streaming_options_{ streaming_options }
+  {
+  }
+
+  void execute(operations::query_request request, query_stream_component::handler_type&& handler)
+  {
+    const std::string client_context_id =
+      request.client_context_id.value_or(uuid::to_string(uuid::random()));
+    const std::chrono::milliseconds timeout = request.timeout.value_or(default_timeout_);
+
+    http_request http_req{ service_type::query, "POST" };
+    http_req.path = "/query/service";
+
+    // Everything about the request that an error context reports, known before dispatch. The
+    // response-side fields (HTTP status, the service's first error, the endpoint) are filled in as
+    // they become available, so a failure at any stage carries as much detail as the buffered path.
+    // Held behind a shared_ptr, not by value: it carries the encoded request body in
+    // `parameters`, and both the dispatch callback and the synchronous dispatch-failure branch
+    // below need it, so sharing avoids deep-copying that body on every streaming request.
+    auto base_ctx = std::make_shared<error_context::query>();
+    base_ctx->client_context_id = client_context_id;
+    base_ctx->statement = request.statement;
+    base_ctx->method = http_req.method;
+    base_ctx->path = http_req.path;
+
+    try {
+      http_req.body = build_streaming_query_body(request, client_context_id, timeout);
+    } catch (const std::exception&) {
+      // A caller-supplied raw/parameter value that is not valid JSON fails encoding; surface it as
+      // invalid_argument rather than letting the exception escape the public API.
+      base_ctx->ec = errc::common::invalid_argument;
+      return handler({}, std::move(*base_ctx));
+    }
+    base_ctx->parameters = http_req.body;
+    http_req.timeout = timeout;
+    http_req.client_context_id = client_context_id;
+    http_req.is_read_only = request.readonly;
+    http_req.headers["connection"] = "keep-alive";
+    http_req.headers["content-type"] = "application/json";
+    if (request.parent_span) {
+      http_req.parent_span = request.parent_span;
+    }
+
+    auto callback_state = std::make_shared<callback_state_type>(std::move(handler));
+
+    using dispatch_error = std::error_code;
+
+    const bool read_only = request.readonly;
+    auto op = http_.do_http_request(
+      http_req,
+      [self = shared_from_this(), callback_state, timeout, read_only, base_ctx](
+        http_response resp, dispatch_error err) mutable {
+        if (auto ec = to_error_code(err)) {
+          base_ctx->ec = ec;
+          self->invoke(callback_state, {}, std::move(*base_ctx));
+          return;
+        }
+        // The response is in hand, so the context can now name the endpoint and HTTP status.
+        base_ctx->http_status = resp.status_code();
+        const auto dispatch_info = resp.dispatch_info();
+        base_ctx->hostname = dispatch_info.hostname;
+        base_ctx->port = dispatch_info.port;
+        if (!dispatch_info.remote_address.empty()) {
+          base_ctx->last_dispatched_to = dispatch_info.remote_address;
+        }
+        if (!dispatch_info.local_address.empty()) {
+          base_ctx->last_dispatched_from = dispatch_info.local_address;
+        }
+        // Use the cluster's streaming tuning, but bound the server between reads with this
+        // request's timeout: if it stalls mid-body the stream times out rather than hanging next()
+        // forever (the request timeout is otherwise released once headers arrive).
+        auto options = self->streaming_options_;
+        options.idle_timeout = timeout;
+        options.is_read_only = read_only;
+        auto stream =
+          std::make_shared<query_stream>(self->io_, resp.body(), options, resp.status_code());
+        stream->start(
+          [self, callback_state, stream, base_ctx](std::error_code early_error) mutable {
+            if (early_error) {
+              // The preamble is what carried the failure, so stamp its diagnostics before tearing
+              // the stream down.
+              base_ctx->ec = early_error;
+              apply_error_details(*base_ctx, stream->error_details());
+              // No consumer will ever take ownership of this stream, so tear the underlying body
+              // (socket + timers) down now rather than when the last handle drops.
+              stream->cancel();
+              self->invoke(callback_state, {}, std::move(*base_ctx));
+              return;
+            }
+            // Hand the base context to the consumer alongside the stream: a terminal error reached
+            // later is stamped onto it, so a mid-stream failure reports the same detail as this
+            // one.
+            self->invoke(callback_state, std::move(*stream), std::move(*base_ctx));
+          });
+      });
+
+    if (!op.has_value()) {
+      base_ctx->ec = to_error_code(op.error());
+      invoke(callback_state, {}, std::move(*base_ctx));
+    }
+  }
+
+private:
+  struct callback_state_type {
+    explicit callback_state_type(query_stream_component::handler_type h)
+      : handler{ std::move(h) }
+    {
+    }
+
+    query_stream_component::handler_type handler;
+    std::mutex mutex{};
+    bool invoked{ false };
+  };
+
+  void invoke(const std::shared_ptr<callback_state_type>& state,
+              query_stream stream,
+              error_context::query ctx)
+  {
+    query_stream_component::handler_type handler;
+    {
+      const std::scoped_lock lock{ state->mutex };
+      if (state->invoked) {
+        return;
+      }
+      state->invoked = true;
+      handler = std::move(state->handler);
+    }
+    if (handler) {
+      handler(std::move(stream), std::move(ctx));
+    }
+  }
+
+  asio::io_context& io_;
+  http_component http_;
+  std::chrono::milliseconds default_timeout_;
+  row_streamer_options streaming_options_;
+};
+
+query_stream_component::query_stream_component(asio::io_context& io,
+                                               http_component http,
+                                               std::chrono::milliseconds default_timeout,
+                                               row_streamer_options streaming_options)
+  : impl_{ std::make_shared<query_stream_component_impl>(io,
+                                                         std::move(http),
+                                                         default_timeout,
+                                                         streaming_options) }
+{
+}
+
+void
+query_stream_component::execute(operations::query_request request, handler_type&& handler) const
+{
+  impl_->execute(std::move(request), std::move(handler));
+}
+} // namespace couchbase::core

data/ext/couchbase/core/query_stream_component.hxx

@@ -0,0 +1,68 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *   Copyright 2026. Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include "error_context/query.hxx"
+#include "operations/document_query.hxx"
+#include "query_stream.hxx"
+#include "utils/movable_function.hxx"
+
+#include <chrono>
+#include <memory>
+#include <system_error>
+
+namespace asio
+{
+class io_context;
+} // namespace asio
+
+namespace couchbase::core
+{
+class http_component;
+
+/**
+ * Dispatches a classic N1QL query as a streaming HTTP request and resolves to a query_stream
+ * handle once the response preamble (signature + upfront errors) has been parsed.
+ *
+ * Unlike the buffered query() path this never materialises the full response body. It performs no
+ * client-side retry or prepared-statement caching: the streaming path is only taken for adhoc
+ * requests (the public layer routes prepared statements to the buffered path).
+ */
+class query_stream_component
+{
+public:
+  // The context carries the error code (error_context::query::ec) alongside the diagnostics the
+  // buffered path reports — statement, parameters, the service's first error, HTTP status and body,
+  // and where the request was dispatched — so a streaming failure is as diagnosable as a buffered
+  // one. It is delivered on both outcomes: when the stream fails to start it describes that
+  // failure, and when it starts it describes the request and its dispatch, for the consumer to
+  // report a terminal error against later. Only `ec` differs between the two: falsy means the
+  // stream started.
+  using handler_type = utils::movable_function<void(query_stream, error_context::query)>;
+
+  query_stream_component(asio::io_context& io,
+                         http_component http,
+                         std::chrono::milliseconds default_timeout,
+                         row_streamer_options streaming_options = {});
+
+  void execute(operations::query_request request, handler_type&& handler) const;
+
+private:
+  std::shared_ptr<class query_stream_component_impl> impl_;
+};
+} // namespace couchbase::core

data/ext/couchbase/core/range_scan_load_balancer.cxx

@@ -36,7 +36,7 @@ range_scan_node_state::range_scan_node_state(std::queue<std::uint16_t> vbuckets)
 auto
 range_scan_node_state::fetch_vbucket_id() -> std::optional<std::uint16_t>
 {
-  const std::lock_guard<std::mutex> lock{ mutex_ };
+  const std::scoped_lock<std::mutex> lock{ mutex_ };
   if (pending_vbuckets_.empty()) {
     return {};
   }
@@ -49,28 +49,28 @@ range_scan_node_state::fetch_vbucket_id() -> std::optional<std::uint16_t>
 void
 range_scan_node_state::notify_stream_ended()
 {
-  const std::lock_guard<std::mutex> lock{ mutex_ };
+  const std::scoped_lock<std::mutex> lock{ mutex_ };
   active_stream_count_--;
 }
 
 void
 range_scan_node_state::enqueue_vbucket(std::uint16_t vbucket_id)
 {
-  const std::lock_guard<std::mutex> lock{ mutex_ };
+  const std::scoped_lock<std::mutex> lock{ mutex_ };
   pending_vbuckets_.push(vbucket_id);
 }
 
 auto
 range_scan_node_state::active_stream_count() -> std::uint16_t
 {
-  const std::lock_guard<std::mutex> lock{ mutex_ };
+  const std::scoped_lock<std::mutex> lock{ mutex_ };
   return active_stream_count_;
 }
 
 auto
 range_scan_node_state::pending_vbucket_count() -> std::size_t
 {
-  const std::lock_guard<std::mutex> lock{ mutex_ };
+  const std::scoped_lock<std::mutex> lock{ mutex_ };
   return pending_vbuckets_.size();
 }
 
@@ -81,8 +81,15 @@ range_scan_load_balancer::range_scan_load_balancer(
 {
   std::map<std::int16_t, std::queue<std::uint16_t>> node_to_vbucket_map{};
   for (std::size_t vbucket_id = 0; vbucket_id < vbucket_map.size(); vbucket_id++) {
-    auto node_id = vbucket_map[vbucket_id][0];
-    node_to_vbucket_map[node_id].push(gsl::narrow_cast<std::uint16_t>(vbucket_id));
+    const auto& chain = vbucket_map[vbucket_id];
+    if (chain.empty()) {
+      // A row that names no node at all cannot be scanned, and there is no node
+      // to queue it against. range_scan_orchestrator refuses such a map before
+      // it starts any stream, so leaving the vbucket out here cannot silently
+      // shorten a scan.
+      continue;
+    }
+    node_to_vbucket_map[chain[0]].push(gsl::narrow_cast<std::uint16_t>(vbucket_id));
   }
   for (auto [node_id, vbucket_ids] : node_to_vbucket_map) {
     nodes_.emplace(node_id, std::move(vbucket_ids));
@@ -98,7 +105,7 @@ range_scan_load_balancer::seed(std::uint64_t seed)
 auto
 range_scan_load_balancer::select_vbucket() -> std::optional<std::uint16_t>
 {
-  const std::lock_guard<std::mutex> lock{ select_vbucket_mutex_ };
+  const std::scoped_lock<std::mutex> lock{ select_vbucket_mutex_ };
 
   auto min_stream_count = std::numeric_limits<std::uint16_t>::max();
   std::optional<std::int16_t> selected_node_id{};

data/ext/couchbase/core/range_scan_orchestrator.cxx

@@ -31,6 +31,7 @@
 
 #include <gsl/util>
 
+#include <algorithm>
 #include <atomic>
 #include <chrono>
 #include <future>
@@ -398,6 +399,16 @@ public:
       return cb(errc::common::invalid_argument, {});
     }
 
+    // A scan needs an active copy for every vbucket, and every row of the map is
+    // indexed below to find one. A row that names no node at all leaves the scan
+    // unable to cover its vbucket, which is reported rather than served as a
+    // silently short result.
+    if (std::any_of(vbucket_map_.begin(), vbucket_map_.end(), [](const auto& chain) {
+          return chain.empty();
+        })) {
+      return cb(errc::network::configuration_not_available, {});
+    }
+
     const get_collection_id_options get_cid_options{ options_.retry_strategy,
                                                      options_.timeout,
                                                      options_.parent_span };
@@ -429,7 +440,8 @@ public:
           };
 
           // Get the active node for the vbucket (values in vbucket map are the active node id
-          // followed by the ids of the replicas)
+          // followed by the ids of the replicas). scan() has already established that every row
+          // names one.
           auto node_id = self->vbucket_map_[vbucket][0];
 
           auto stream = std::make_shared<range_scan_stream>(
@@ -510,7 +522,7 @@ public:
           } else {
             // Empty signal means that stream has completed
             {
-              const std::lock_guard<std::mutex> lock{ self->stream_map_mutex_ };
+              const std::scoped_lock<std::mutex> lock{ self->stream_map_mutex_ };
               self->streams_.erase(signal.vbucket_id);
             }
             return asio::post(asio::bind_executor(
@@ -540,7 +552,7 @@ public:
       auto v = vbucket_id.value();
       std::shared_ptr<range_scan_stream> stream{};
       {
-        const std::lock_guard<std::mutex> lock{ stream_map_mutex_ };
+        const std::scoped_lock<std::mutex> lock{ stream_map_mutex_ };
         stream = streams_.at(v);
       }
       CB_LOG_TRACE("scanning vbucket {} at node {}", vbucket_id.value(), stream->node_id());

data/ext/couchbase/core/response_handler.hxx

@@ -18,28 +18,65 @@
 #pragma once
 
 #include "core/error_context/key_value_error_map_info.hxx"
+
 #include <couchbase/retry_reason.hxx>
 
 #include <memory>
 #include <optional>
 #include <system_error>
 
+namespace couchbase::core::mcbp
+{
+class queue_request;
+} // namespace couchbase::core::mcbp
+
 namespace couchbase::core
 {
 namespace io
 {
 struct mcbp_message;
+class mcbp_session;
 } // namespace io
-namespace mcbp
-{
-class queue_request;
-} // namespace mcbp
 
+/**
+ * Interface for handling MCBP (Memcached Binary Protocol) responses dispatched by
+ * @ref io::mcbp_session.
+ *
+ * Implementors receive the decoded response together with enough context to populate
+ * dispatch tracing spans and key-value error contexts without holding a long-lived
+ * reference to the session.
+ *
+ * @note Adding, removing, or reordering parameters of @ref handle_response is an ABI
+ *       breaking change for any out-of-tree implementors of this interface.
+ */
 class response_handler
 {
 public:
+  response_handler() = default;
+  response_handler(const response_handler&) = default;
+  response_handler(response_handler&&) = delete;
+  auto operator=(const response_handler&) -> response_handler& = default;
+  auto operator=(response_handler&&) -> response_handler& = delete;
   virtual ~response_handler() = default;
+
+  /**
+   * Called by @ref io::mcbp_session when a queued request completes (successfully or with
+   * an error).
+   *
+   * @param request  The originating request.  May be empty when the session is shutting
+   *                 down and synthetically cancels in-flight operations.
+   * @param session  A snapshot of the session that delivered the response, carrying
+   *                 connection metadata (remote/local address and port) needed to populate
+   *                 dispatch tracing spans and error contexts.  Used by
+   *                 @ref crud_component (CXXCBC-798) to build
+   *                 @ref key_value_error_context without retaining a live session reference.
+   * @param error    Transport-level or protocol-level error code; empty on success.
+   * @param reason   Retry classification hint supplied by the session.
+   * @param msg      The raw decoded MCBP response message.
+   * @param error_info  Optional extended error information from the server's error map.
+   */
   virtual void handle_response(std::shared_ptr<mcbp::queue_request> request,
+                               std::optional<io::mcbp_session> session,
                                std::error_code error,
                                retry_reason reason,
                                io::mcbp_message msg,