couchbase 3.8.2 → 3.9.0

data/ext/couchbase/core/operations/document_analytics.cxx

@@ -16,10 +16,12 @@
  */
 
 #include "document_analytics.hxx"
+#include "analytics_response_parsing.hxx"
 #include "core/cluster_options.hxx"
 #include "core/logger/logger.hxx"
-#include "core/utils/duration_parser.hxx"
+#include "core/logger/redaction.hxx"
 #include "core/utils/json.hxx"
+#include "core/utils/name_codec.hxx"
 
 #include <couchbase/error_codes.hxx>
 
@@ -28,15 +30,10 @@
 
 namespace couchbase::core::operations
 {
-auto
-analytics_request::encode_to(analytics_request::encoded_request_type& encoded,
-                             http_context& context) -> std::error_code
+void
+encode_analytics_options(tao::json::value& body, const analytics_request& request)
 {
-  tao::json::value body{ { "statement", statement },
-                         { "client_context_id", encoded.client_context_id },
-                         { "timeout", fmt::format("{}ms", encoded.timeout.count()) } };
-
-  for (const auto& [name, value] : named_parameters) {
+  for (const auto& [name, value] : request.named_parameters) {
     Expects(name.empty() == false);
     std::string key = name;
     if (key[0] != '$') {
@@ -44,19 +41,19 @@ analytics_request::encode_to(analytics_request::encoded_request_type& encoded,
     }
     body[key] = utils::json::parse(value);
   }
-  if (!positional_parameters.empty()) {
+  if (!request.positional_parameters.empty()) {
     std::vector<tao::json::value> parameters;
-    parameters.reserve(positional_parameters.size());
-    for (const auto& value : positional_parameters) {
+    parameters.reserve(request.positional_parameters.size());
+    for (const auto& value : request.positional_parameters) {
       parameters.emplace_back(utils::json::parse(value));
     }
     body["args"] = std::move(parameters);
   }
-  if (readonly) {
+  if (request.readonly) {
     body["readonly"] = true;
   }
-  if (scan_consistency) {
-    switch (scan_consistency.value()) {
+  if (request.scan_consistency) {
+    switch (request.scan_consistency.value()) {
       case couchbase::core::analytics_scan_consistency::not_bounded:
         body["scan_consistency"] = "not_bounded";
         break;
@@ -65,14 +62,30 @@ analytics_request::encode_to(analytics_request::encoded_request_type& encoded,
         break;
     }
   }
-  if (scope_qualifier) {
-    body["query_context"] = scope_qualifier;
-  } else if (scope_name && bucket_name) {
-    body["query_context"] = fmt::format("default:`{}`.`{}`", *bucket_name, *scope_name);
+  if (request.scope_qualifier) {
+    body["query_context"] = request.scope_qualifier.value();
+  } else if (request.scope_name && request.bucket_name) {
+    // The query context is parsed as SQL++, so its two names are quoted through the same encoder
+    // the management statements use rather than wrapped in bare backticks.
+    body["query_context"] = fmt::format("default:{}.{}",
+                                        utils::analytics::quote_identifier(*request.bucket_name),
+                                        utils::analytics::quote_identifier(*request.scope_name));
   }
-  for (const auto& [name, value] : raw) {
+  for (const auto& [name, value] : request.raw) {
     body[name] = utils::json::parse(value);
   }
+}
+
+auto
+analytics_request::encode_to(analytics_request::encoded_request_type& encoded,
+                             http_context& context) -> std::error_code
+{
+  tao::json::value body{ { "statement", statement },
+                         { "client_context_id", encoded.client_context_id },
+                         { "timeout", fmt::format("{}ms", encoded.timeout.count()) } };
+
+  encode_analytics_options(body, *this);
+
   encoded.type = type;
   encoded.headers["content-type"] = "application/json";
   if (priority) {
@@ -85,11 +98,11 @@ analytics_request::encode_to(analytics_request::encoded_request_type& encoded,
   if (context.options.show_queries) {
     CB_LOG_INFO("ANALYTICS: client_context_id=\"{}\", {}",
                 encoded.client_context_id,
-                utils::json::generate(body["statement"]));
+                logger::user_data(utils::json::generate(body["statement"])));
   } else {
     CB_LOG_DEBUG("ANALYTICS: client_context_id=\"{}\", {}",
                  encoded.client_context_id,
-                 utils::json::generate(body["statement"]));
+                 logger::user_data(utils::json::generate(body["statement"])));
   }
   if (row_callback) {
     encoded.streaming.emplace(couchbase::core::io::streaming_settings{
@@ -116,78 +129,12 @@ analytics_request::make_response(error_context::analytics&& ctx,
       response.ctx.ec = errc::common::parsing_failure;
       return response;
     }
-    if (const auto* i = payload.find("requestID"); i != nullptr) {
-      response.meta.request_id = i->get_string();
-    }
-    if (const auto* i = payload.find("clientContextID"); i != nullptr) {
-      response.meta.client_context_id = i->get_string();
-      if (response.ctx.client_context_id != response.meta.client_context_id) {
-        CB_LOG_WARNING(R"(unexpected clientContextID returned by service: "{}", expected "{}")",
-                       response.meta.client_context_id,
-                       response.ctx.client_context_id);
-      }
-    }
-    if (auto& status_prop = payload.at("status"); status_prop.is_string()) {
-      const auto& status = status_prop.get_string();
-      if (status == "running") {
-        response.meta.status = analytics_response::analytics_status::running;
-      } else if (status == "success") {
-        response.meta.status = analytics_response::analytics_status::success;
-      } else if (status == "errors") {
-        response.meta.status = analytics_response::analytics_status::errors;
-      } else if (status == "completed") {
-        response.meta.status = analytics_response::analytics_status::completed;
-      } else if (status == "stopped") {
-        response.meta.status = analytics_response::analytics_status::stopped;
-      } else if (status == "timedout") {
-        response.meta.status = analytics_response::analytics_status::timedout;
-      } else if (status == "closed") {
-        response.meta.status = analytics_response::analytics_status::closed;
-      } else if (status == "fatal") {
-        response.meta.status = analytics_response::analytics_status::fatal;
-      } else if (status == "aborted") {
-        response.meta.status = analytics_response::analytics_status::aborted;
-      } else {
-        response.meta.status = analytics_response::analytics_status::unknown;
-      }
-    } else {
-      response.meta.status = analytics_response::analytics_status::unknown;
-    }
-
-    if (const auto* s = payload.find("signature"); s != nullptr) {
-      response.meta.signature = couchbase::core::utils::json::generate(*s);
-    }
-
-    if (const auto* metrics = payload.find("metrics"); metrics != nullptr) {
-      response.meta.metrics.result_count = metrics->at("resultCount").get_unsigned();
-      response.meta.metrics.result_size = metrics->at("resultSize").get_unsigned();
-      response.meta.metrics.elapsed_time =
-        utils::parse_duration(metrics->at("elapsedTime").get_string());
-      response.meta.metrics.execution_time =
-        utils::parse_duration(metrics->at("executionTime").get_string());
-      response.meta.metrics.processed_objects = metrics->at("processedObjects").get_unsigned();
-      response.meta.metrics.error_count =
-        metrics->optional<std::uint64_t>("errorCount").value_or(0);
-      response.meta.metrics.warning_count =
-        metrics->optional<std::uint64_t>("warningCount").value_or(0);
-    }
-
-    if (const auto* e = payload.find("errors"); e != nullptr) {
-      for (const auto& err : e->get_array()) {
-        couchbase::core::operations::analytics_response::analytics_problem problem;
-        problem.code = err.at("code").get_unsigned();
-        problem.message = err.at("msg").get_string();
-        response.meta.errors.emplace_back(problem);
-      }
-    }
-
-    if (const auto* w = payload.find("warnings"); w != nullptr) {
-      for (const auto& warn : w->get_array()) {
-        couchbase::core::operations::analytics_response::analytics_problem problem;
-        problem.code = warn.at("code").get_unsigned();
-        problem.message = warn.at("msg").get_string();
-        response.meta.warnings.emplace_back(problem);
-      }
+    response.meta = parse_analytics_meta(payload);
+    if (response.ctx.client_context_id != response.meta.client_context_id &&
+        !response.meta.client_context_id.empty()) {
+      CB_LOG_WARNING(R"(unexpected clientContextID returned by service: "{}", expected "{}")",
+                     response.meta.client_context_id,
+                     response.ctx.client_context_id);
     }
 
     if (const auto* r = payload.find("results"); r != nullptr) {
@@ -197,43 +144,14 @@ analytics_request::make_response(error_context::analytics&& ctx,
       }
     }
 
-    if (response.meta.status != analytics_response::analytics_status::success) {
-      response.ctx.first_error_code = response.meta.errors.front().code;
-      response.ctx.first_error_message = response.meta.errors.front().message;
-      switch (response.ctx.first_error_code) {
-        case 21002: /* Request timed out and will be cancelled */
-          response.ctx.ec = errc::common::unambiguous_timeout;
-          break;
-        case 23007: /* Job queue is full with [string] jobs */
-          response.ctx.ec = errc::analytics::job_queue_full;
-          break;
-        case 24044: /* Cannot find dataset [string] because there is no dataverse declared, nor an
-                       alias with name [string]! */
-        case 24045: /* Cannot find dataset [string] in dataverse [string] nor an alias with name
-                       [string]! */
-        case 24025: /* Cannot find dataset with name [string] in dataverse [string] */
-          response.ctx.ec = errc::analytics::dataset_not_found;
-          break;
-        case 24034: /* Cannot find dataverse with name [string] */
-          response.ctx.ec = errc::analytics::dataverse_not_found;
-          break;
-        case 24040: /* A dataset with name [string] already exists in dataverse [string] */
-          response.ctx.ec = errc::analytics::dataset_exists;
-          break;
-        case 24039: /* A dataverse with this name [string] already exists. */
-          response.ctx.ec = errc::analytics::dataverse_exists;
-          break;
-        case 24006: /* Link [string] does not exist | Link [string] does not exist */
-          response.ctx.ec = errc::analytics::link_not_found;
-          break;
-        default:
-          if (response.ctx.first_error_code >= 24000 && response.ctx.first_error_code < 25000) {
-            response.ctx.ec = errc::analytics::compilation_failure;
-          }
-      }
-      if (!response.ctx.ec) {
-        response.ctx.ec = errc::common::internal_server_failure;
+    // RFC-58: a 400 is a failure whatever status the body reports.
+    if (response.meta.status != analytics_response::analytics_status::success ||
+        encoded.status_code == 400) {
+      if (!response.meta.errors.empty()) {
+        response.ctx.first_error_code = response.meta.errors.front().code;
+        response.ctx.first_error_message = response.meta.errors.front().message;
       }
+      response.ctx.ec = map_analytics_error(response.meta, encoded.status_code);
     }
   }
   return response;

data/ext/couchbase/core/operations/document_analytics.hxx

@@ -27,8 +27,20 @@
 #include "core/public_fwd.hxx"
 #include "core/timeout_defaults.hxx"
 
+#include <tao/json/forward.hpp>
+
 namespace couchbase::core::operations
 {
+struct analytics_request;
+
+/**
+ * Encodes the shared analytics request-body options (parameters, readonly, consistency,
+ * query_context, raw) common to the buffered and streaming analytics paths onto @p body, so both
+ * paths stay byte-for-byte consistent.
+ */
+void
+encode_analytics_options(tao::json::value& body, const analytics_request& request);
+
 struct analytics_response {
   struct analytics_metrics {
     std::chrono::nanoseconds elapsed_time{};
@@ -101,8 +113,8 @@ struct analytics_request {
   std::optional<std::string> client_context_id{};
   std::optional<std::chrono::milliseconds> timeout{};
 
-  [[nodiscard]] auto encode_to(encoded_request_type& encoded,
-                               http_context& context) -> std::error_code;
+  [[nodiscard]] auto encode_to(encoded_request_type& encoded, http_context& context)
+    -> std::error_code;
   [[nodiscard]] auto make_response(error_context::analytics&& ctx,
                                    const encoded_response_type& encoded) const
     -> analytics_response;

data/ext/couchbase/core/operations/document_get_all_replicas.hxx

@@ -20,16 +20,21 @@
 #include <couchbase/error_codes.hxx>
 #include <couchbase/node_id.hxx>
 
+#include "core/document_id_redaction.hxx"
 #include "core/error_context/key_value.hxx"
-#include "core/impl/get_replica.hxx"
 #include "core/impl/replica_utils.hxx"
 #include "core/logger/logger.hxx"
+#include "core/logger/redaction.hxx"
 #include "core/operations/document_get.hxx"
+#include "core/operations/document_get_replica.hxx"
 #include "core/operations/operation_traits.hxx"
 #include "core/public_fwd.hxx"
+#include "core/service_type.hxx"
 #include "core/tracing/constants.hxx"
+#include "core/utils/deadline.hxx"
 #include "core/utils/movable_function.hxx"
 
+#include <chrono>
 #include <memory>
 #include <mutex>
 
@@ -64,11 +69,14 @@ struct get_all_replicas_request {
   template<typename Core, typename Handler>
   void execute(Core core, Handler handler)
   {
+    const auto deadline = std::chrono::steady_clock::now() +
+                          timeout.value_or(core->default_timeout_for(service_type::key_value));
     core->with_bucket_configuration(
       id.bucket(),
+      deadline,
       [core,
        id = id,
-       timeout = timeout,
+       deadline,
        read_preference = read_preference,
        parent_span = std::move(parent_span),
        h = std::forward<Handler>(handler)](
@@ -86,8 +94,8 @@ struct get_all_replicas_request {
         if (nodes.empty()) {
           CB_LOG_DEBUG(
             "Unable to retrieve replicas for \"{}\", server_group={}, number_of_replicas={}",
-            id,
-            origin.options().server_group,
+            logger::document(id),
+            logger::metadata(origin.options().server_group),
             config->num_replicas.value_or(0));
           return h(response_type{
             make_key_value_error_context(errc::key_value::document_irretrievable, id) });
@@ -129,9 +137,9 @@ struct get_all_replicas_request {
             document_id replica_id{ id };
             replica_id.node_index(node.index);
             core->execute(
-              impl::get_replica_request{
+              operations::get_replica_request{
                 std::move(replica_id),
-                timeout,
+                utils::remaining_until(deadline),
                 {},
                 {},
                 {},
@@ -184,7 +192,7 @@ struct get_all_replicas_request {
                 document_id{ id },
                 {},
                 {},
-                timeout,
+                utils::remaining_until(deadline),
                 {},
                 subop_span,
               },

data/ext/couchbase/core/operations/document_get_any_replica.hxx

@@ -17,15 +17,20 @@
 
 #pragma once
 
+#include "core/document_id_redaction.hxx"
 #include "core/error_context/key_value.hxx"
-#include "core/impl/get_replica.hxx"
 #include "core/impl/replica_utils.hxx"
 #include "core/impl/with_cancellation.hxx"
+#include "core/logger/redaction.hxx"
 #include "core/operations/document_get.hxx"
+#include "core/operations/document_get_replica.hxx"
 #include "core/operations/operation_traits.hxx"
+#include "core/service_type.hxx"
+#include "core/utils/deadline.hxx"
 #include "core/utils/movable_function.hxx"
 #include "couchbase/error_codes.hxx"
 
+#include <chrono>
 #include <functional>
 #include <memory>
 #include <mutex>
@@ -57,17 +62,23 @@ struct get_any_replica_request {
   template<typename Core, typename Handler>
   void execute(Core core, Handler handler)
   {
+    const auto deadline = std::chrono::steady_clock::now() +
+                          timeout.value_or(core->default_timeout_for(service_type::key_value));
     core->with_bucket_configuration(
       id.bucket(),
+      deadline,
       [core,
        id = id,
-       timeout = timeout,
+       deadline,
        read_preference = read_preference,
        parent_span = std::move(parent_span),
        h = std::forward<Handler>(handler)](
         std::error_code ec, std::shared_ptr<topology::configuration> config) mutable {
+        if (ec) {
+          return h(response_type{ make_key_value_error_context(ec, id) });
+        }
         const auto [e, origin] = core->origin();
-        if (e && !ec) {
+        if (e) {
           ec = e;
         }
 
@@ -76,8 +87,8 @@ struct get_any_replica_request {
         if (nodes.empty()) {
           CB_LOG_DEBUG(
             "Unable to retrieve replicas for \"{}\", server_group={}, number_of_replicas={}",
-            id,
-            origin.options().server_group,
+            logger::document(id),
+            logger::metadata(origin.options().server_group),
             config->num_replicas.value_or(0));
           ec = errc::key_value::document_irretrievable;
         }
@@ -137,10 +148,10 @@ struct get_any_replica_request {
           if (node.is_replica) {
             document_id replica_id{ id };
             replica_id.node_index(node.index);
-            impl::with_cancellation<impl::get_replica_request> req{
+            impl::with_cancellation<operations::get_replica_request> req{
               {
                 std::move(replica_id),
-                timeout,
+                utils::remaining_until(deadline),
                 {},
                 {},
                 {},
@@ -191,7 +202,7 @@ struct get_any_replica_request {
                 id,
                 {},
                 {},
-                timeout,
+                utils::remaining_until(deadline),
                 {},
                 subop_span,
               },

data/ext/couchbase/core/operations/document_get_projected.cxx

@@ -79,7 +79,7 @@ subdoc_lookup(tao::json::value& root, const std::string& path) -> std::optional<
 void
 subdoc_apply_projection(tao::json::value& root,
                         const std::string& path,
-                        tao::json::value& value,
+                        const tao::json::value& value,
                         bool preserve_array_indexes)
 {
   std::string::size_type offset = 0;

data/ext/couchbase/core/{impl/get_replica.cxx → operations/document_get_replica.cxx}

@@ -15,16 +15,34 @@
  *   limitations under the License.
  */
 
-#include "get_replica.hxx"
+#include "core/operations/document_get_replica.hxx"
 
 #include "core/error_context/key_value_error_context.hxx"
 #include "core/io/mcbp_context.hxx"
 
+#include <optional>
 #include <system_error>
 #include <utility>
 
-namespace couchbase::core::impl
+namespace couchbase::core::operations
 {
+auto
+get_replica_request::resolve_route(const topology::configuration& config) const
+  -> std::optional<impl::replica_route_decision>
+{
+  // The replica fan-out callers share this type but leave the strategy unset:
+  // they pin the replica through id.node_index() and want the vbucket-map
+  // routing the dispatch path already applies.
+  if (!selection.has_value()) {
+    return {};
+  }
+
+  // Named vbucket rather than partition: the request carries a partition member,
+  // and MSVC rejects a local that hides it.
+  const auto vbucket = config.map_key(id.key(), 0).first;
+  return impl::resolve_replica_index(config, vbucket, selection->replica_index, selection->wrap);
+}
+
 auto
 get_replica_request::encode_to(get_replica_request::encoded_request_type& encoded,
                                core::mcbp_context&& /* context */) const -> std::error_code
@@ -48,4 +66,4 @@ get_replica_request::make_response(key_value_error_context&& ctx,
   }
   return response;
 }
-} // namespace couchbase::core::impl
+} // namespace couchbase::core::operations

data/ext/couchbase/core/{impl/get_replica.hxx → operations/document_get_replica.hxx}

@@ -19,14 +19,19 @@
 
 #include "core/error_context/key_value_error_context.hxx"
 
+#include "core/impl/replica_utils.hxx"
 #include "core/io/mcbp_context.hxx"
 #include "core/io/retry_context.hxx"
+#include "core/operations/operation_traits.hxx"
 #include "core/protocol/client_request.hxx"
 #include "core/protocol/cmd_get_replica.hxx"
 #include "core/public_fwd.hxx"
 #include "core/timeout_defaults.hxx"
+#include "core/utils/movable_function.hxx"
 
-namespace couchbase::core::impl
+#include <couchbase/get_replica_result.hxx>
+
+namespace couchbase::core::operations
 {
 struct get_replica_response {
   key_value_error_context ctx{};
@@ -50,12 +55,34 @@ struct get_replica_request {
   std::uint32_t opaque{};
   io::retry_context<true> retries{};
   std::shared_ptr<couchbase::tracing::request_span> parent_span{ nullptr };
+  /**
+   * Absent for the replica fan-out callers, which pin the replica through
+   * @c id.node_index(). When this is unset resolve_route() returns an empty
+   * optional, declining to route, and the dispatch path falls back to the
+   * vbucket-map routing that honours @c id.node_index().
+   */
+  std::optional<impl::replica_selection> selection{};
+
+  /**
+   * Selects the node this request should be sent to, against the topology of
+   * the current attempt. Empty when the request wants the vbucket-map routing
+   * the dispatch path applies to every other request.
+   */
+  [[nodiscard]] auto resolve_route(const topology::configuration& config) const
+    -> std::optional<impl::replica_route_decision>;
 
-  [[nodiscard]] auto encode_to(encoded_request_type& encoded,
-                               core::mcbp_context&& context) const -> std::error_code;
+  [[nodiscard]] auto encode_to(encoded_request_type& encoded, core::mcbp_context&& context) const
+    -> std::error_code;
 
   [[nodiscard]] auto make_response(key_value_error_context&& ctx,
                                    const encoded_response_type& encoded) const
     -> get_replica_response;
 };
-} // namespace couchbase::core::impl
+
+using movable_get_replica_handler =
+  utils::movable_function<void(couchbase::error, get_replica_result)>;
+
+template<>
+struct resolves_own_route<get_replica_request> : public std::true_type {
+};
+} // namespace couchbase::core::operations

data/ext/couchbase/core/operations/document_lookup_in.cxx

@@ -66,7 +66,6 @@ lookup_in_request::make_response(key_value_error_context&& ctx,
   bool deleted = false;
   couchbase::cas cas{};
   std::vector<lookup_in_response::entry> fields{};
-  std::error_code ec = ctx.ec();
   std::optional<std::size_t> first_error_index{};
   std::optional<std::string> first_error_path{};
 
@@ -90,13 +89,14 @@ lookup_in_request::make_response(key_value_error_context&& ctx,
                                                static_cast<std::uint16_t>(res_entry.status));
       if (fields[i].opcode == protocol::subdoc_opcode::exists &&
           fields[i].ec == errc::key_value::path_not_found) {
+        // exists treats path_not_found as success ("false"); suppress the error so the
+        // exists flag below carries the result instead.
         fields[i].ec.clear();
       }
-      if (!fields[i].ec && !ctx.ec()) {
-        ec = fields[i].ec;
-      }
-      if (!first_error_index && !fields[i].ec) {
-        first_error_index = i;
+      // Use original_index, not wire-order i: XATTR specs are reordered during encode_to(),
+      // so i != original_index (was the CXXCBC-788 bug).
+      if (!first_error_index && fields[i].ec) {
+        first_error_index = fields[i].original_index;
         first_error_path = fields[i].path;
       }
       fields[i].exists = res_entry.status == key_value_status_code::success ||
@@ -107,16 +107,16 @@ lookup_in_request::make_response(key_value_error_context&& ctx,
         fields[i].value = utils::to_binary(res_entry.value);
       }
     }
-    if (!ec) {
-      cas = encoded.cas();
-    }
+    // Always assign CAS; previously guarded by a local ec that was polluted by per-path
+    // errors, zeroing CAS whenever any path failed (CXXCBC-788).
+    cas = encoded.cas();
     std::sort(fields.begin(), fields.end(), [](const auto& lhs, const auto& rhs) {
       return lhs.original_index < rhs.original_index;
     });
   }
 
   return lookup_in_response{
-    make_subdocument_error_context(ctx, ec, first_error_path, first_error_index, deleted),
+    make_subdocument_error_context(ctx, ctx.ec(), first_error_path, first_error_index, deleted),
     cas,
     std::move(fields),
     deleted,