couchbase 3.8.2 → 3.9.1

data/ext/couchbase/core/impl/public_query_stream_result.cxx

@@ -0,0 +1,380 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *   Copyright 2026. Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include <couchbase/error_codes.hxx>
+#include <couchbase/query_meta_data.hxx>
+#include <couchbase/query_metrics.hxx>
+#include <couchbase/query_status.hxx>
+#include <couchbase/query_stream_result.hxx>
+#include <couchbase/query_warning.hxx>
+
+#include "core/operations/document_query.hxx"
+#include "core/utils/binary.hxx"
+
+#include "error.hxx"
+#include "internal_query_stream_result.hxx"
+#include "observability_recorder.hxx"
+#include "query.hxx"
+
+#include <future>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <system_error>
+#include <utility>
+#include <vector>
+
+namespace couchbase
+{
+namespace
+{
+auto
+to_binary(const std::string& s) -> codec::binary
+{
+  return core::utils::to_binary(s);
+}
+
+auto
+build_query_meta_data(core::operations::query_response::query_meta_data meta) -> query_meta_data
+{
+  std::vector<query_warning> warnings;
+  if (meta.warnings) {
+    warnings.reserve(meta.warnings->size());
+    for (auto& w : *meta.warnings) {
+      warnings.emplace_back(w.code, std::move(w.message), w.reason, w.retry);
+    }
+  }
+
+  std::optional<query_metrics> metrics;
+  if (meta.metrics) {
+    metrics = query_metrics{
+      meta.metrics->elapsed_time, meta.metrics->execution_time, meta.metrics->result_count,
+      meta.metrics->result_size,  meta.metrics->sort_count,     meta.metrics->mutation_count,
+      meta.metrics->error_count,  meta.metrics->warning_count,
+    };
+  }
+
+  std::optional<codec::binary> signature;
+  if (meta.signature) {
+    signature = to_binary(*meta.signature);
+  }
+
+  std::optional<codec::binary> profile;
+  if (meta.profile) {
+    profile = to_binary(*meta.profile);
+  }
+
+  return query_meta_data{
+    std::move(meta.request_id),
+    std::move(meta.client_context_id),
+    core::impl::map_query_status(std::move(meta.status)),
+    std::move(warnings),
+    metrics,
+    std::move(signature),
+    std::move(profile),
+  };
+}
+} // namespace
+
+// ---------------------------------------------------------------------------
+// internal_query_stream_result implementation
+// ---------------------------------------------------------------------------
+
+internal_query_stream_result::internal_query_stream_result(
+  core::query_stream stream,
+  std::unique_ptr<core::impl::observability_recorder> obs_rec,
+  core::error_context::query ctx)
+  : stream_{ std::move(stream) }
+  , ctx_{ std::move(ctx) }
+  , obs_rec_{ std::move(obs_rec) }
+{
+}
+
+internal_query_stream_result::~internal_query_stream_result()
+{
+  cancel();
+}
+
+void
+internal_query_stream_result::cancel()
+{
+  // Resolve any parked meta_data() futures with a cancellation error so they don't throw
+  // broken_promise when the stream is torn down before reaching a natural terminal. Idempotent
+  // via terminal_reached_, so a real terminal that already ran is preserved.
+  resolve_meta_data(errc::common::request_canceled);
+  stream_.cancel();
+}
+
+auto
+internal_query_stream_result::signature() const -> std::optional<codec::binary>
+{
+  auto raw = stream_.signature();
+  if (!raw) {
+    return {};
+  }
+  return to_binary(*raw);
+}
+
+void
+internal_query_stream_result::resolve_meta_data(std::error_code ec)
+{
+  std::pair<error, query_meta_data> value;
+  if (ec) {
+    // Report the terminal against the request's context with whatever the response said stamped on,
+    // so a mid-stream failure carries the same detail (statement, service error, endpoint) as one
+    // that never started. A cancellation leaves the service error code and message unset, because
+    // the service never reported one, but the envelope parsed so far is still reported as the body:
+    // it names the request id the service logs are keyed by, and the buffered path likewise reports
+    // whatever body it received.
+    auto ctx = ctx_;
+    ctx.ec = ec;
+    core::apply_error_details(ctx, stream_.error_details());
+    value.first = core::impl::make_error(ctx);
+  }
+  auto core_meta = stream_.meta_data();
+  if (core_meta) {
+    value.second = build_query_meta_data(std::move(*core_meta));
+  }
+
+  std::vector<std::promise<std::pair<error, query_meta_data>>> to_satisfy;
+  {
+    const std::scoped_lock lk{ meta_mutex_ };
+    if (terminal_reached_) {
+      return;
+    }
+    terminal_reached_ = true;
+    terminal_value_ = value;
+    to_satisfy = std::move(pending_meta_promises_);
+    pending_meta_promises_.clear();
+  }
+  for (auto& promise : to_satisfy) {
+    promise.set_value(value);
+  }
+  // Reached exactly once (the terminal_reached_ guard above returns on every later call), so the
+  // operation is finished here: record the latency metric and end the operation span. ec is the
+  // terminal error (falsy on a clean drain, request_canceled on teardown before a natural end).
+  if (obs_rec_) {
+    obs_rec_->finish(ec);
+  }
+}
+
+auto
+internal_query_stream_result::meta_data() -> std::future<std::pair<error, query_meta_data>>
+{
+  const std::scoped_lock lk{ meta_mutex_ };
+  std::promise<std::pair<error, query_meta_data>> promise;
+  auto future = promise.get_future();
+  if (terminal_reached_) {
+    promise.set_value(terminal_value_);
+  } else {
+    pending_meta_promises_.push_back(std::move(promise));
+  }
+  return future;
+}
+
+void
+internal_query_stream_result::next(query_row_handler&& handler)
+{
+  {
+    // Idempotent terminal: once the stream has ended, re-deliver the terminal (clean end or the
+    // terminal error) instead of issuing a pull that would block forever on the drained channel.
+    const std::scoped_lock lk{ meta_mutex_ };
+    if (terminal_reached_) {
+      return handler(terminal_value_.first, std::nullopt);
+    }
+  }
+  bool expected = false;
+  if (!pull_in_flight_.compare_exchange_strong(expected, true)) {
+    // Enforce the single-outstanding-next() contract instead of racing the single-consumer channel.
+    return handler(
+      error{ errc::common::invalid_argument, "a previous next() call is still in flight" },
+      std::nullopt);
+  }
+  // Capture a strong self so the object outlives an in-flight pull; dropping the public handle
+  // mid-pull would otherwise destroy this object before the channel completion runs (use-after-
+  // free). The stream is always owned via shared_ptr (make_shared in dispatch_query_stream).
+  stream_.next_row([handler = std::move(handler), self = shared_from_this()](
+                     std::optional<std::string> row, std::error_code ec) mutable {
+    if (row.has_value()) {
+      // Real row (ec is falsy). Release the pull gate before invoking the handler so it may
+      // immediately issue the next pull.
+      self->pull_in_flight_.store(false);
+      return handler({}, query_row{ to_binary(*row) });
+    }
+    // Terminal (nullopt row): publish the terminal state and intentionally leave pull_in_flight_
+    // set. Every later next() is served by the terminal_reached_ guard (idempotent re-delivery) and
+    // must never issue another pull onto the drained channel, so the gate is never reopened — this
+    // also backstops a caller that violates the single-outstanding-next() contract from a racing
+    // thread before terminal_reached_ becomes visible.
+    self->resolve_meta_data(ec);
+    if (ec) {
+      // Deliver the contextual error resolve_meta_data() has just published rather than building a
+      // bare one here. This is the first and usually only observation of a mid-stream terminal, so
+      // it has to carry the same diagnostics as every later next() (served by the terminal_reached_
+      // guard above) and as meta_data(); constructing a fresh error here would report the one case
+      // this class exists to describe with an empty context.
+      error terminal_error;
+      {
+        const std::scoped_lock lk{ self->meta_mutex_ };
+        terminal_error = self->terminal_value_.first;
+      }
+      return handler(std::move(terminal_error), {});
+    }
+    return handler({}, {});
+  });
+}
+
+// ---------------------------------------------------------------------------
+// query_stream_result public implementation
+// ---------------------------------------------------------------------------
+
+query_stream_result::query_stream_result(std::shared_ptr<internal_query_stream_result> internal)
+  : internal_{ std::move(internal) }
+{
+}
+
+void
+query_stream_result::next(query_row_handler&& handler) const
+{
+  if (!internal_) {
+    // Empty handle behaves as an immediately-exhausted stream.
+    handler(error{}, std::nullopt);
+    return;
+  }
+  return internal_->next(std::move(handler));
+}
+
+auto
+query_stream_result::next() const -> std::future<std::pair<error, std::optional<query_row>>>
+{
+  auto barrier = std::make_shared<std::promise<std::pair<error, std::optional<query_row>>>>();
+  if (!internal_) {
+    // Empty handle behaves as an immediately-exhausted stream.
+    barrier->set_value({ error{}, std::nullopt });
+    return barrier->get_future();
+  }
+  internal_->next([barrier](const auto& err, const auto& row) mutable {
+    barrier->set_value({ err, row });
+  });
+  return barrier->get_future();
+}
+
+auto
+query_stream_result::signature() const -> std::optional<codec::binary>
+{
+  if (!internal_) {
+    return {};
+  }
+  return internal_->signature();
+}
+
+auto
+query_stream_result::meta_data() const -> std::future<std::pair<error, query_meta_data>>
+{
+  if (!internal_) {
+    // Empty handle resolves immediately with empty metadata.
+    std::promise<std::pair<error, query_meta_data>> p;
+    p.set_value({ error{}, query_meta_data{} });
+    return p.get_future();
+  }
+  return internal_->meta_data();
+}
+
+void
+query_stream_result::cancel() const
+{
+  if (internal_) {
+    return internal_->cancel();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// iterator implementation
+// ---------------------------------------------------------------------------
+
+query_stream_result::iterator::iterator(std::shared_ptr<internal_query_stream_result> internal)
+  : internal_{ std::move(internal) }
+{
+  if (internal_) {
+    fetch_item();
+  } else {
+    // An empty result handle has no rows: the iterator starts already at end.
+    terminal_ = true;
+  }
+}
+
+void
+query_stream_result::iterator::fetch_item()
+{
+  auto barrier = std::make_shared<std::promise<std::pair<error, std::optional<query_row>>>>();
+  internal_->next([barrier](const error& err, std::optional<query_row> row) mutable {
+    barrier->set_value({ err, std::move(row) });
+  });
+  auto result = barrier->get_future().get();
+  if (result.first) {
+    // Terminal error: deliver it as one visible element (still != end()) so a range-for loop can
+    // observe it; the next increment then advances to true end.
+    item_ = { result.first, {} };
+    terminal_ = false;
+    error_pending_ = true;
+  } else if (!result.second.has_value()) {
+    // Clean end-of-stream.
+    item_ = {};
+    terminal_ = true;
+    error_pending_ = false;
+  } else {
+    item_ = { {}, *result.second };
+    terminal_ = false;
+    error_pending_ = false;
+  }
+}
+
+auto
+query_stream_result::iterator::operator*() const -> std::pair<error, query_row>
+{
+  return item_;
+}
+
+auto
+query_stream_result::iterator::operator++() -> query_stream_result::iterator&
+{
+  if (error_pending_) {
+    // The terminal error element has been consumed; advance to end().
+    error_pending_ = false;
+    item_ = {};
+    terminal_ = true;
+    return *this;
+  }
+  fetch_item();
+  return *this;
+}
+
+auto
+query_stream_result::begin() const -> query_stream_result::iterator
+{
+  // The iterator constructor treats a null internal handle as an already-drained stream, so an
+  // empty result handle yields begin() == end().
+  return query_stream_result::iterator{ internal_ };
+}
+
+auto
+query_stream_result::end() const -> query_stream_result::end_sentinel
+{
+  return {};
+}
+} // namespace couchbase

data/ext/couchbase/core/impl/public_transaction_get_multi_replicas_from_preferred_server_group_result.cxx

@@ -0,0 +1,42 @@
+/*
+ *     Copyright 2021-Present Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include <couchbase/transactions/transaction_get_multi_replicas_from_preferred_server_group_result.hxx>
+
+#include <couchbase/codec/encoded_value.hxx>
+#include <couchbase/error_codes.hxx>
+
+#include <cstddef>
+#include <stdexcept>
+#include <string>
+#include <system_error>
+
+namespace couchbase::transactions
+{
+auto
+transaction_get_multi_replicas_from_preferred_server_group_result::content_at(
+  std::size_t spec_index) const -> const codec::encoded_value&
+{
+  if (spec_index >= content_.size()) {
+    throw std::invalid_argument("spec index " + std::to_string(spec_index) + " is not valid");
+  }
+  if (const auto& content = content_[spec_index]; content.has_value()) {
+    return content.value();
+  }
+  throw std::system_error(errc::key_value::document_not_found,
+                          "document was not found for index " + std::to_string(spec_index));
+}
+} // namespace couchbase::transactions

data/ext/couchbase/core/impl/public_transaction_get_multi_result.cxx

@@ -0,0 +1,42 @@
+/*
+ *     Copyright 2021-Present Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#include <couchbase/transactions/transaction_get_multi_result.hxx>
+
+#include <couchbase/codec/encoded_value.hxx>
+#include <couchbase/error_codes.hxx>
+
+#include <cstddef>
+#include <stdexcept>
+#include <string>
+#include <system_error>
+
+namespace couchbase::transactions
+{
+auto
+transaction_get_multi_result::content_at(std::size_t spec_index) const
+  -> const codec::encoded_value&
+{
+  if (spec_index >= content_.size()) {
+    throw std::invalid_argument("spec index " + std::to_string(spec_index) + " is not valid");
+  }
+  if (const auto& content = content_[spec_index]; content.has_value()) {
+    return content.value();
+  }
+  throw std::system_error(errc::key_value::document_not_found,
+                          "document was not found for index " + std::to_string(spec_index));
+}
+} // namespace couchbase::transactions

data/ext/couchbase/core/impl/query.cxx

@@ -18,22 +18,31 @@
 #include <couchbase/error_codes.hxx>
 #include <couchbase/query_options.hxx>
 #include <couchbase/transactions/transaction_query_result.hxx>
+
+#include <algorithm>
+#include <cctype>
 #include <utility>
 
 #include "core/cluster.hxx"
 #include "core/error_context/transaction_op_error_context.hxx"
+#include "core/free_form_http_request.hxx"
 #include "core/operations/document_query.hxx"
+#include "core/query_stream.hxx"
 #include "core/utils/binary.hxx"
+#include "error.hxx"
+#include "internal_query_stream_result.hxx"
+#include "observability_recorder.hxx"
+#include "query.hxx"
+
+#include <memory>
 
 namespace couchbase::core::impl
 {
-namespace
-{
 auto
-map_status(std::string status) -> query_status
+map_query_status(std::string status) -> query_status
 {
   std::transform(status.cbegin(), status.cend(), status.begin(), [](unsigned char c) {
-    return std::tolower(c);
+    return static_cast<char>(std::tolower(c));
   });
   if (status == "running") {
     return query_status::running;
@@ -65,8 +74,10 @@ map_status(std::string status) -> query_status
   return query_status::unknown;
 }
 
+namespace
+{
 auto
-map_rows(operations::query_response& resp) -> std::vector<codec::binary>
+map_rows(const operations::query_response& resp) -> std::vector<codec::binary>
 {
   std::vector<codec::binary> rows;
   rows.reserve(resp.rows.size());
@@ -154,7 +165,7 @@ build_result(operations::query_response& resp) -> query_result
     query_meta_data{
       std::move(resp.meta.request_id),
       std::move(resp.meta.client_context_id),
-      map_status(resp.meta.status),
+      map_query_status(resp.meta.status),
       map_warnings(resp),
       map_metrics(resp),
       map_signature(resp),
@@ -201,6 +212,64 @@ build_query_request(std::string statement,
   return request;
 }
 
+void
+dispatch_query_stream(const core::cluster& core,
+                      core::operations::query_request request,
+                      std::unique_ptr<observability_recorder> obs_rec,
+                      query_stream_handler&& handler)
+{
+  if (!request.adhoc) {
+    // Prepared statements rely on buffered retry/replay, so they cannot be streamed live. Run the
+    // buffered path and replay the already-parsed result through a buffered query_stream — no JSON
+    // re-serialization or re-parsing — so callers observe identical semantics to the adhoc path.
+    auto& io = core.io_context();
+    core.execute(std::move(request),
+                 [&io, obs_rec = std::move(obs_rec), handler = std::move(handler)](
+                   operations::query_response resp) mutable {
+                   if (resp.ctx.ec) {
+                     // Terminal reached before a handle exists: record the operation metric/span
+                     // here.
+                     obs_rec->finish(resp.ctx.ec);
+                     return handler(make_error(resp.ctx), {});
+                   }
+                   auto stream = std::make_shared<core::query_stream>(
+                     io, std::move(resp.rows), std::move(resp.meta));
+                   stream->start([obs_rec = std::move(obs_rec),
+                                  handler = std::move(handler),
+                                  stream,
+                                  ctx = std::move(resp.ctx)](std::error_code early_error) mutable {
+                     if (early_error) {
+                       obs_rec->finish(early_error);
+                       ctx.ec = early_error;
+                       return handler(make_error(ctx), {});
+                     }
+                     // Transfer the recorder to the handle; it finishes the operation at the stream
+                     // terminal. The buffered request's context travels with it, so a terminal
+                     // error is reported with the same detail the buffered path would have given.
+                     auto internal = std::make_shared<internal_query_stream_result>(
+                       std::move(*stream), std::move(obs_rec), std::move(ctx));
+                     handler({}, query_stream_result{ std::move(internal) });
+                   });
+                 });
+    return;
+  }
+
+  core.query_stream(std::move(request),
+                    [obs_rec = std::move(obs_rec), handler = std::move(handler)](
+                      core::query_stream stream, core::error_context::query ctx) mutable {
+                      if (ctx.ec) {
+                        obs_rec->finish(ctx.ec);
+                        return handler(make_error(ctx), {});
+                      }
+                      // The context travels with the handle: a terminal error reached later is
+                      // reported with the same statement/endpoint detail as a failure to start,
+                      // plus whatever the trailer said.
+                      auto internal = std::make_shared<internal_query_stream_result>(
+                        std::move(stream), std::move(obs_rec), std::move(ctx));
+                      handler({}, query_stream_result{ std::move(internal) });
+                    });
+}
+
 auto
 build_transaction_query_result(operations::query_response resp,
                                std::error_code txn_ec /*defaults to 0*/)
@@ -222,7 +291,7 @@ build_transaction_query_result(operations::query_response resp,
     { query_meta_data{
         std::move(resp.meta.request_id),
         std::move(resp.meta.client_context_id),
-        map_status(resp.meta.status),
+        map_query_status(resp.meta.status),
         map_warnings(resp),
         map_metrics(resp),
         map_signature(resp),

data/ext/couchbase/core/impl/query.hxx

@@ -20,9 +20,27 @@
 #include "core/operations/document_query.hxx"
 
 #include <couchbase/query_options.hxx>
+#include <couchbase/query_status.hxx>
+
+#include <memory>
+#include <string>
+
+namespace couchbase::core
+{
+class cluster;
+} // namespace couchbase::core
 
 namespace couchbase::core::impl
 {
+class observability_recorder;
+
+/**
+ * Maps the N1QL `status` string from a query response onto the public query_status enum. Shared by
+ * the buffered result builder and the streaming result handle so there is one source of truth.
+ */
+auto
+map_query_status(std::string status) -> query_status;
+
 auto
 build_query_request(std::string statement,
                     std::optional<std::string> query_context,
@@ -32,4 +50,22 @@ build_query_request(std::string statement,
 
 auto
 build_result(operations::query_response& resp) -> query_result;
+
+/**
+ * Dispatches a query as a streaming request and resolves the handler with a query_stream_result.
+ *
+ * Adhoc requests take the lazy streaming path. Prepared statements (request.adhoc == false) are
+ * not streamed; they fall back to the buffered query() path and their rows are replayed through an
+ * in-memory query_stream so callers observe identical semantics.
+ *
+ * @param obs_rec the operation's observability recorder. Its operation span is already threaded
+ * into request.parent_span by the caller; ownership is transferred to the resulting stream handle,
+ * which calls finish() once when the stream reaches its terminal (drained, errored, or cancelled),
+ * so the streaming path emits the same operation span + latency metric as the buffered query().
+ */
+void
+dispatch_query_stream(const core::cluster& core,
+                      core::operations::query_request request,
+                      std::unique_ptr<observability_recorder> obs_rec,
+                      query_stream_handler&& handler);
 } // namespace couchbase::core::impl