couchbase 3.8.1 → 3.9.0

data/ext/couchbase/core/transactions/attempt_context_impl.cxx

@@ -42,6 +42,7 @@
 #include "internal/utils.hxx"
 #include "staged_mutation.hxx"
 
+#include <cassert>
 #include <optional>
 
 namespace couchbase::core::transactions
@@ -200,11 +201,14 @@ attempt_context_impl::check_and_handle_blocking_transactions(
       if (err) {
         return cb(err);
       }
-      return check_atr_entry_for_blocking_document(doc,
-                                                   exp_delay(std::chrono::milliseconds(50),
-                                                             std::chrono::milliseconds(500),
-                                                             std::chrono::seconds(1)),
-                                                   std::move(cb));
+      return check_atr_entry_for_blocking_document(
+        doc,
+        std::make_shared<async_timed_exp_delay>(
+          std::make_shared<asio::steady_timer>(cluster_ref().io_context()),
+          std::chrono::milliseconds(50),
+          std::chrono::milliseconds(500),
+          std::chrono::seconds(1)),
+        std::move(cb));
     }
     CB_ATTEMPT_CTX_LOG_DEBUG(this,
                              "doc \"{}\" is in another transaction {}, but doesn't have enough "
@@ -233,10 +237,12 @@ attempt_context_impl::get(const core::document_id& id) -> transaction_get_result
 void
 attempt_context_impl::get(const core::document_id& id, Callback&& cb)
 {
-  if (op_list_.get_mode().is_query()) {
-    return get_with_query(id, false, std::move(cb));
-  }
   cache_error_async(cb, [self = shared_from_this(), id, cb]() mutable {
+    // Mode is read after cache_error_async incremented the op count, closing the
+    // race with set_query_mode() -- see the comment in insert() for the rationale.
+    if (self->op_list_.get_mode().is_query()) {
+      return self->get_with_query(id, false, std::move(cb));
+    }
     self->check_if_done(cb);
     self->do_get(
       id,
@@ -323,10 +329,12 @@ void
 attempt_context_impl::get_optional(const core::document_id& id, Callback&& cb)
 {
 
-  if (op_list_.get_mode().is_query()) {
-    return get_with_query(id, true, std::move(cb));
-  }
   cache_error_async(cb, [self = shared_from_this(), id, cb]() mutable {
+    // Mode is read after cache_error_async incremented the op count, closing the
+    // race with set_query_mode() -- see the comment in insert() for the rationale.
+    if (self->op_list_.get_mode().is_query()) {
+      return self->get_with_query(id, true, std::move(cb));
+    }
     self->ensure_open_bucket(
       id.bucket(), [self, id, cb = std::move(cb)](std::error_code ec) mutable {
         if (ec) {
@@ -886,11 +894,13 @@ attempt_context_impl::replace(const transaction_get_result& document,
                               Callback&& cb)
 {
 
-  if (op_list_.get_mode().is_query()) {
-    return replace_raw_with_query(document, std::move(content), std::move(cb));
-  }
   return cache_error_async(
     cb, [self = shared_from_this(), cb, document, content = std::move(content)]() mutable {
+      // Mode is read after cache_error_async incremented the op count, closing the
+      // race with set_query_mode() -- see the comment in insert() for the rationale.
+      if (self->op_list_.get_mode().is_query()) {
+        return self->replace_raw_with_query(document, std::move(content), std::move(cb));
+      }
       self->ensure_open_bucket(
         document.bucket(),
         [self, cb = std::move(cb), document, content = std::move(content)](
@@ -912,7 +922,7 @@ attempt_context_impl::replace(const transaction_get_result& document,
             CB_ATTEMPT_CTX_LOG_TRACE(
               self, "replacing {} with {}", document, utils::to_string(content.data));
             self->check_if_done(cb);
-            staged_mutation* existing_sm = self->staged_mutations_->find_any(document.id());
+            const staged_mutation* existing_sm = self->staged_mutations_->find_any(document.id());
             if (existing_sm != nullptr && existing_sm->type() == staged_mutation_type::REMOVE) {
               CB_ATTEMPT_CTX_LOG_DEBUG(
                 self, "found existing REMOVE of {} while replacing", document);
@@ -965,9 +975,7 @@ attempt_context_impl::replace(const transaction_get_result& document,
                       self->create_staged_insert(document.id(),
                                                  std::move(content),
                                                  document.cas().value(),
-                                                 exp_delay(std::chrono::milliseconds(5),
-                                                           std::chrono::milliseconds(300),
-                                                           self->overall()->config().timeout),
+                                                 self->staged_insert_delay(),
                                                  op_id,
                                                  std::move(cb));
                       return;
@@ -1026,6 +1034,13 @@ attempt_context_impl::create_staged_replace(
   const std::optional<document_metadata>& document_metadata,
   Handler&& cb)
 {
+  // ExtThreadSafety invariant -- see create_staged_insert() for the full rationale and
+  // the spec reference (couchbase-transactions-specs/transactions-extensions.md,
+  // "ExtThreadSafety").  replace() reads the mode after this op is counted, so reaching
+  // this staging point in query mode means that ordering has regressed.
+  assert(!op_list_.query_mode_entered() &&
+         "ExtThreadSafety: KV replace staged after query mode entered -- would be missing "
+         "from BEGIN WORK and lost at commit");
   operations::mutate_in_request req{ id };
   const bool binary =
     codec::codec_flags::has_common_flags(content.flags, codec::codec_flags::binary_common_flags);
@@ -1274,11 +1289,20 @@ attempt_context_impl::insert(const core::document_id& id,
                              codec::encoded_value content,
                              Callback&& cb)
 {
-  if (op_list_.get_mode().is_query()) {
-    return insert_raw_with_query(id, std::move(content), std::move(cb));
-  }
   return cache_error_async(
     cb, [self = shared_from_this(), id, cb, content = std::move(content)]() mutable {
+      // The op count was incremented by cache_error_async before this lambda runs.
+      // Reading the mode here -- after the increment -- closes a race with
+      // set_query_mode(): a query switching to query mode first waits for all
+      // in-flight KV ops to drain before issuing BEGIN WORK, so once this op is
+      // counted, either we observe KV mode and our staged mutation is guaranteed to
+      // be part of BEGIN WORK, or we observe query mode and route through query.
+      // Checking the mode before the increment (as the code used to) left a window
+      // where a query could flip the mode in the gap, after which this op would
+      // stage a KV mutation that BEGIN WORK had already missed -- silently losing it.
+      if (self->op_list_.get_mode().is_query()) {
+        return self->insert_raw_with_query(id, std::move(content), std::move(cb));
+      }
       self->ensure_open_bucket(
         id.bucket(),
         [self, id, content = std::move(content), cb = std::move(cb)](std::error_code ec) mutable {
@@ -1289,7 +1313,7 @@ attempt_context_impl::insert(const core::document_id& id,
           try {
             self->check_if_done(cb);
             auto op_id = uid_generator::next();
-            staged_mutation* existing_sm = self->staged_mutations_->find_any(id);
+            const staged_mutation* existing_sm = self->staged_mutations_->find_any(id);
             if ((existing_sm != nullptr) &&
                 (existing_sm->type() == staged_mutation_type::INSERT ||
                  existing_sm->type() == staged_mutation_type::REPLACE)) {
@@ -1322,14 +1346,8 @@ attempt_context_impl::insert(const core::document_id& id,
                                                      std::move(cb));
                 }
                 const std::uint64_t cas = 0;
-                self->create_staged_insert(id,
-                                           std::move(content),
-                                           cas,
-                                           exp_delay(std::chrono::milliseconds(5),
-                                                     std::chrono::milliseconds(300),
-                                                     self->overall()->config().timeout),
-                                           op_id,
-                                           std::move(cb));
+                self->create_staged_insert(
+                  id, std::move(content), cas, self->staged_insert_delay(), op_id, std::move(cb));
               });
           } catch (const std::exception& e) {
             return self->op_completed_with_error(
@@ -1378,19 +1396,24 @@ attempt_context_impl::select_atr_if_needed_unlocked(
     CB_ATTEMPT_CTX_LOG_ERROR(this, "unexpected error \"{}\" during select atr if needed", e.what());
   }
 }
-template<typename Handler, typename Delay>
+template<typename Handler>
 void
-attempt_context_impl::check_atr_entry_for_blocking_document(const transaction_get_result& doc,
-                                                            Delay delay,
-                                                            Handler&& cb)
+attempt_context_impl::check_atr_entry_for_blocking_document(
+  const transaction_get_result& doc,
+  std::shared_ptr<async_timed_exp_delay> delay,
+  Handler&& cb)
 {
-  try {
-    delay();
-    return hooks_.before_check_atr_entry_for_blocking_doc(
-      shared_from_this(),
+  (*delay)([self = shared_from_this(), doc, delay, cb = std::forward<Handler>(cb)](
+             const std::exception_ptr& delay_err) mutable {
+    if (delay_err) {
+      return cb(transaction_operation_failed(FAIL_WRITE_WRITE_CONFLICT,
+                                             "document is in another transaction")
+                  .retry());
+    }
+    return self->hooks_.before_check_atr_entry_for_blocking_doc(
+      self,
       doc.id().key(),
-      [self = shared_from_this(), delay = std::move(delay), cb = std::forward<Handler>(cb), doc](
-        auto ec) mutable {
+      [self, delay = std::move(delay), cb = std::move(cb), doc](auto ec) mutable {
         if (ec) {
           return cb(transaction_operation_failed(FAIL_WRITE_WRITE_CONFLICT,
                                                  "document is in another transaction")
@@ -1430,7 +1453,8 @@ attempt_context_impl::check_atr_entry_for_blocking_document(const transaction_ge
                                                "existing atr entry found in state {}, retrying",
                                                attempt_state_name(it->state()));
                   }
-                  return self->check_atr_entry_for_blocking_document(doc, delay, std::move(cb));
+                  return self->check_atr_entry_for_blocking_document(
+                    doc, std::move(delay), std::move(cb));
                 }
               }
               CB_ATTEMPT_CTX_LOG_DEBUG(self, "no blocking atr entry");
@@ -1442,20 +1466,18 @@ attempt_context_impl::check_atr_entry_for_blocking_document(const transaction_ge
                         .retry());
           });
       });
-  } catch (const retry_operation_timeout&) {
-    return cb(
-      transaction_operation_failed(FAIL_WRITE_WRITE_CONFLICT, "document is in another transaction")
-        .retry());
-  }
+  });
 }
 void
 attempt_context_impl::remove(const transaction_get_result& document, VoidCallback&& cb)
 {
 
-  if (op_list_.get_mode().is_query()) {
-    return remove_with_query(document, std::move(cb));
-  }
   return cache_error_async(cb, [self = shared_from_this(), document, cb]() mutable {
+    // Mode is read after cache_error_async incremented the op count, closing the
+    // race with set_query_mode() -- see the comment in insert() for the rationale.
+    if (self->op_list_.get_mode().is_query()) {
+      return self->remove_with_query(document, std::move(cb));
+    }
     self->check_if_done(cb);
     self->ensure_open_bucket(
       document.bucket(), [self, document, cb = std::move(cb)](std::error_code ec) mutable {
@@ -1539,6 +1561,15 @@ attempt_context_impl::remove(const transaction_get_result& document, VoidCallbac
                       return error_handler(
                         *ec, "before_staged_remove hook raised error", std::move(cb));
                     }
+                    // ExtThreadSafety invariant -- see create_staged_insert() for the
+                    // full rationale and spec reference
+                    // (couchbase-transactions-specs/transactions-extensions.md,
+                    // "ExtThreadSafety").  remove() reads the mode after this op is
+                    // counted, so reaching this staging point in query mode means that
+                    // ordering has regressed.
+                    assert(!self->op_list_.query_mode_entered() &&
+                           "ExtThreadSafety: KV remove staged after query mode entered -- "
+                           "would be missing from BEGIN WORK and lost at commit");
                     CB_ATTEMPT_CTX_LOG_TRACE(self,
                                              "about to remove doc {} with cas {}",
                                              document.id(),
@@ -1959,11 +1990,11 @@ dump_request(const core::operations::query_request& req) -> std::string
   std::string raw = "{";
   for (const auto& x : req.raw) {
     raw += x.first;
-    raw += ":";
+    raw += ':';
     raw += x.second.str();
-    raw += ",";
+    raw += ',';
   }
-  raw += "}";
+  raw += '}';
   std::string params;
   for (const auto& x : req.positional_parameters) {
     params.append(x.str());
@@ -2015,6 +2046,7 @@ attempt_context_impl::wrap_query(
 
   if (check_expiry) {
     if (has_expired_client_side(hook_point, std::nullopt)) {
+      query_check_expired_ = true;
       auto err = std::make_exception_ptr(
         transaction_operation_failed(FAIL_EXPIRY,
                                      fmt::format("{} expired in stage {}", statement, hook_point))
@@ -2203,59 +2235,69 @@ make_kv_txdata(std::optional<transaction_get_result> doc = std::nullopt) -> tao:
 void
 attempt_context_impl::get_with_query(const core::document_id& id, bool optional, Callback&& cb)
 {
-  cache_error_async(cb, [self = shared_from_this(), id, optional, cb]() mutable {
-    couchbase::transactions::transaction_query_options opts;
-    opts.readonly(true);
-    return self->wrap_query(
-      KV_GET,
-      opts,
-      make_params(id, {}),
-      make_kv_txdata(),
-      STAGE_QUERY_KV_GET,
-      true,
-      {},
-      [self, id, optional, cb = std::move(cb)](std::exception_ptr err,
-                                               core::operations::query_response resp) mutable {
-        if (resp.ctx.ec == couchbase::errc::key_value::document_not_found) {
+  // Called only from get()/get_optional(), already within their cache_error_async()
+  // (op counted, existing_error() checked); see insert_raw_with_query() for why we do
+  // not re-wrap.
+  auto self = shared_from_this();
+  couchbase::transactions::transaction_query_options opts;
+  opts.readonly(true);
+  return self->wrap_query(
+    KV_GET,
+    opts,
+    make_params(id, {}),
+    make_kv_txdata(),
+    STAGE_QUERY_KV_GET,
+    true,
+    {},
+    [self, id, optional, cb = std::move(cb)](std::exception_ptr err,
+                                             core::operations::query_response resp) mutable {
+      if (resp.ctx.ec == couchbase::errc::key_value::document_not_found) {
+        // A non-optional get() must fail rather than report an empty result: its callback contract
+        // (and the synchronous get() wrapper) dereferences the returned optional, so err==nullptr
+        // with res==nullopt would be a null dereference. Mirror the rows-empty handling below and
+        // the KV get() path.
+        if (optional) {
           return self->op_completed_with_callback(std::move(cb),
                                                   std::optional<transaction_get_result>());
         }
-        if (!err) {
-          // make a transaction_get_result from the row...
-          try {
-            if (resp.rows.empty()) {
-              if (optional) {
-                return self->op_completed_with_callback(std::move(cb),
-                                                        std::optional<transaction_get_result>());
-              }
-              return self->op_completed_with_error(
-                std::move(cb),
-                transaction_operation_failed(FAIL_DOC_NOT_FOUND, "document not found"));
+        return self->op_completed_with_error(
+          std::move(cb), transaction_operation_failed(FAIL_DOC_NOT_FOUND, "document not found"));
+      }
+      if (!err) {
+        // make a transaction_get_result from the row...
+        try {
+          if (resp.rows.empty()) {
+            if (optional) {
+              return self->op_completed_with_callback(std::move(cb),
+                                                      std::optional<transaction_get_result>());
             }
-            CB_ATTEMPT_CTX_LOG_TRACE(self, "get_with_query got: {}", resp.rows.front());
-            transaction_get_result doc(id, core::utils::json::parse(resp.rows.front()));
-            return self->op_completed_with_callback(std::move(cb),
-                                                    std::optional<transaction_get_result>(doc));
-          } catch (const std::exception& e) {
-            // TODO(SA): unsure what to do here, but this is pretty fatal, so
             return self->op_completed_with_error(
-              std::move(cb), transaction_operation_failed(FAIL_OTHER, e.what()));
+              std::move(cb),
+              transaction_operation_failed(FAIL_DOC_NOT_FOUND, "document not found"));
           }
+          CB_ATTEMPT_CTX_LOG_TRACE(self, "get_with_query got: {}", resp.rows.front());
+          transaction_get_result doc(id, core::utils::json::parse(resp.rows.front()));
+          return self->op_completed_with_callback(std::move(cb),
+                                                  std::optional<transaction_get_result>(doc));
+        } catch (const std::exception& e) {
+          // Unexpected error here: fail with FAIL_OTHER so the transaction rolls back.
+          return self->op_completed_with_error(std::move(cb),
+                                               transaction_operation_failed(FAIL_OTHER, e.what()));
         }
-        // for get_optional.   <sigh>
-        if (optional) {
-          try {
-            std::rethrow_exception(err);
-          } catch (const document_not_found&) {
-            return self->op_completed_with_callback(std::move(cb),
-                                                    std::optional<transaction_get_result>());
-          } catch (...) {
-            return self->op_completed_with_error(std::move(cb), std::current_exception());
-          }
+      }
+      // for get_optional.   <sigh>
+      if (optional) {
+        try {
+          std::rethrow_exception(err);
+        } catch (const document_not_found&) {
+          return self->op_completed_with_callback(std::move(cb),
+                                                  std::optional<transaction_get_result>());
+        } catch (...) {
+          return self->op_completed_with_error(std::move(cb), std::current_exception());
         }
-        return self->op_completed_with_error(std::move(cb), std::move(err));
-      });
-  });
+      }
+      return self->op_completed_with_error(std::move(cb), std::move(err));
+    });
 }
 
 void
@@ -2263,46 +2305,55 @@ attempt_context_impl::insert_raw_with_query(const core::document_id& id,
                                             codec::encoded_value content,
                                             Callback&& cb)
 {
-  cache_error_async(
-    cb, [self = shared_from_this(), id, content = std::move(content), cb]() mutable {
-      const couchbase::transactions::transaction_query_options opts;
-      return self->wrap_query(
-        KV_INSERT,
-        opts,
-        make_params(id, std::move(content)),
-        make_kv_txdata(),
-        STAGE_QUERY_KV_INSERT,
-        true,
-        {},
-        [self, id, cb = std::move(cb)](const std::exception_ptr& err,
-                                       core::operations::query_response resp) mutable {
-          if (err) {
-            try {
-              std::rethrow_exception(err);
-            } catch (const transaction_operation_failed& e) {
-              return self->op_completed_with_error(std::move(cb), e);
-            } catch (const op_exception& ex) {
-              return self->op_completed_with_error(std::move(cb), ex);
-            } catch (const std::exception& e) {
-              return self->op_completed_with_error(
-                std::move(cb), transaction_operation_failed(FAIL_OTHER, e.what()));
-            } catch (...) {
-              return self->op_completed_with_error(
-                std::move(cb), transaction_operation_failed(FAIL_OTHER, "unexpected error"));
-            }
-          }
-          // make a transaction_get_result from the row...
-          try {
-            CB_ATTEMPT_CTX_LOG_TRACE(self, "insert_raw_with_query got: {}", resp.rows.front());
-            transaction_get_result doc(id, core::utils::json::parse(resp.rows.front()));
-            return self->op_completed_with_callback(std::move(cb),
-                                                    std::optional<transaction_get_result>(doc));
-          } catch (const std::exception& e) {
-            // TODO(SA): unsure what to do here, but this is pretty fatal, so
-            return self->op_completed_with_error(
-              std::move(cb), transaction_operation_failed(FAIL_OTHER, e.what()));
-          }
-        });
+  // Called only from insert(), already within its cache_error_async(): the op has
+  // been counted in the wait-group and existing_error() has been checked, so we must
+  // not wrap again here -- double-counting would leave the op permanently in-flight
+  // and deadlock the commit/rollback wait.
+  auto self = shared_from_this();
+  const couchbase::transactions::transaction_query_options opts;
+  return self->wrap_query(
+    KV_INSERT,
+    opts,
+    make_params(id, std::move(content)),
+    make_kv_txdata(),
+    STAGE_QUERY_KV_INSERT,
+    true,
+    {},
+    [self, id, cb = std::move(cb)](const std::exception_ptr& err,
+                                   core::operations::query_response resp) mutable {
+      if (err) {
+        try {
+          std::rethrow_exception(err);
+        } catch (const transaction_operation_failed& e) {
+          return self->op_completed_with_error(std::move(cb), e);
+        } catch (const op_exception& ex) {
+          return self->op_completed_with_error(std::move(cb), ex);
+        } catch (const std::exception& e) {
+          return self->op_completed_with_error(std::move(cb),
+                                               transaction_operation_failed(FAIL_OTHER, e.what()));
+        } catch (...) {
+          return self->op_completed_with_error(
+            std::move(cb), transaction_operation_failed(FAIL_OTHER, "unexpected error"));
+        }
+      }
+      // make a transaction_get_result from the row...
+      try {
+        if (resp.rows.empty()) {
+          // A successful insert query must return the staged document row; no rows is an
+          // unexpected protocol condition. front() on an empty vector is UB, so guard explicitly.
+          return self->op_completed_with_error(
+            std::move(cb),
+            transaction_operation_failed(FAIL_OTHER, "insert query returned no rows"));
+        }
+        CB_ATTEMPT_CTX_LOG_TRACE(self, "insert_raw_with_query got: {}", resp.rows.front());
+        transaction_get_result doc(id, core::utils::json::parse(resp.rows.front()));
+        return self->op_completed_with_callback(std::move(cb),
+                                                std::optional<transaction_get_result>(doc));
+      } catch (const std::exception& e) {
+        // Unexpected error here: fail with FAIL_OTHER so the transaction rolls back.
+        return self->op_completed_with_error(std::move(cb),
+                                             transaction_operation_failed(FAIL_OTHER, e.what()));
+      }
     });
 }
 
@@ -2311,91 +2362,103 @@ attempt_context_impl::replace_raw_with_query(const transaction_get_result& docum
                                              codec::encoded_value content,
                                              Callback&& cb)
 {
-  cache_error_async(
-    cb, [self = shared_from_this(), document, content = std::move(content), cb]() mutable {
-      const couchbase::transactions::transaction_query_options opts;
-      return self->wrap_query(
-        KV_REPLACE,
-        opts,
-        make_params(document.id(), std::move(content)),
-        make_kv_txdata(document),
-        STAGE_QUERY_KV_REPLACE,
-        true,
-        {},
-        [self, id = document.id(), cb = std::move(cb)](
-          const std::exception_ptr& err, core::operations::query_response resp) mutable {
-          if (err) {
-            try {
-              std::rethrow_exception(err);
-            } catch (const query_cas_mismatch& e) {
-              return self->op_completed_with_error(
-                std::move(cb), transaction_operation_failed(FAIL_CAS_MISMATCH, e.what()).retry());
-            } catch (const document_not_found& e) {
-              return self->op_completed_with_error(
-                std::move(cb), transaction_operation_failed(FAIL_DOC_NOT_FOUND, e.what()).retry());
-            } catch (const transaction_operation_failed& e) {
-              return self->op_completed_with_error(std::move(cb), e);
-            } catch (std::exception& e) {
-              return self->op_completed_with_error(
-                std::move(cb), transaction_operation_failed(FAIL_OTHER, e.what()));
-            } catch (...) {
-              return self->op_completed_with_error(
-                std::move(cb), transaction_operation_failed(FAIL_OTHER, "unexpected exception"));
-            }
-          }
-          // make a transaction_get_result from the row...
-          try {
-            CB_ATTEMPT_CTX_LOG_TRACE(self, "replace_raw_with_query got: {}", resp.rows.front());
-            transaction_get_result doc(id, core::utils::json::parse(resp.rows.front()));
-            return self->op_completed_with_callback(std::move(cb),
-                                                    std::optional<transaction_get_result>(doc));
-          } catch (const std::exception& e) {
-            // TODO(SA): unsure what to do here, but this is pretty fatal, so
-            return self->op_completed_with_error(
-              std::move(cb), transaction_operation_failed(FAIL_OTHER, e.what()));
-          }
-        });
+  // Called only from replace(), already within its cache_error_async() (op counted,
+  // existing_error() checked); see insert_raw_with_query() for why we do not re-wrap.
+  auto self = shared_from_this();
+  const couchbase::transactions::transaction_query_options opts;
+  return self->wrap_query(
+    KV_REPLACE,
+    opts,
+    make_params(document.id(), std::move(content)),
+    make_kv_txdata(document),
+    STAGE_QUERY_KV_REPLACE,
+    true,
+    {},
+    [self, id = document.id(), cb = std::move(cb)](const std::exception_ptr& err,
+                                                   core::operations::query_response resp) mutable {
+      if (err) {
+        try {
+          std::rethrow_exception(err);
+        } catch (const query_cas_mismatch& e) {
+          return self->op_completed_with_error(
+            std::move(cb), transaction_operation_failed(FAIL_CAS_MISMATCH, e.what()).retry());
+        } catch (const document_not_found& e) {
+          return self->op_completed_with_error(
+            std::move(cb), transaction_operation_failed(FAIL_DOC_NOT_FOUND, e.what()).retry());
+        } catch (const transaction_operation_failed& e) {
+          return self->op_completed_with_error(std::move(cb), e);
+        } catch (const op_exception& ex) {
+          return self->op_completed_with_error(std::move(cb), ex);
+        } catch (std::exception& e) {
+          return self->op_completed_with_error(std::move(cb),
+                                               transaction_operation_failed(FAIL_OTHER, e.what()));
+        } catch (...) {
+          return self->op_completed_with_error(
+            std::move(cb), transaction_operation_failed(FAIL_OTHER, "unexpected exception"));
+        }
+      }
+      // make a transaction_get_result from the row...
+      try {
+        if (resp.rows.empty()) {
+          // A successful replace query must return the staged document row; no rows is an
+          // unexpected protocol condition. front() on an empty vector is UB, so guard explicitly.
+          return self->op_completed_with_error(
+            std::move(cb),
+            transaction_operation_failed(FAIL_OTHER, "replace query returned no rows"));
+        }
+        CB_ATTEMPT_CTX_LOG_TRACE(self, "replace_raw_with_query got: {}", resp.rows.front());
+        transaction_get_result doc(id, core::utils::json::parse(resp.rows.front()));
+        return self->op_completed_with_callback(std::move(cb),
+                                                std::optional<transaction_get_result>(doc));
+      } catch (const std::exception& e) {
+        // Unexpected error here: fail with FAIL_OTHER so the transaction rolls back.
+        return self->op_completed_with_error(std::move(cb),
+                                             transaction_operation_failed(FAIL_OTHER, e.what()));
+      }
     });
 }
 
 void
 attempt_context_impl::remove_with_query(const transaction_get_result& document, VoidCallback&& cb)
 {
-  cache_error_async(cb, [self = shared_from_this(), document, cb]() {
-    const couchbase::transactions::transaction_query_options opts;
-    return self->wrap_query(
-      KV_REMOVE,
-      opts,
-      make_params(document.id(), {}),
-      make_kv_txdata(document),
-      STAGE_QUERY_KV_REMOVE,
-      true,
-      {},
-      [self, id = document.id(), cb](const std::exception_ptr& err,
-                                     const core::operations::query_response& /* resp */) mutable {
-        if (err) {
-          try {
-            std::rethrow_exception(err);
-          } catch (const transaction_operation_failed& e) {
-            return self->op_completed_with_error(std::move(cb), e);
-          } catch (const document_not_found& e) {
-            return self->op_completed_with_error(
-              std::move(cb), transaction_operation_failed(FAIL_DOC_NOT_FOUND, e.what()).retry());
-          } catch (const query_cas_mismatch& e) {
-            return self->op_completed_with_error(
-              std::move(cb), transaction_operation_failed(FAIL_CAS_MISMATCH, e.what()).retry());
-          } catch (const std::exception& e) {
-            return self->op_completed_with_error(
-              std::move(cb), transaction_operation_failed(FAIL_OTHER, e.what()));
-          } catch (...) {
-            return self->op_completed_with_error(
-              std::move(cb), transaction_operation_failed(FAIL_OTHER, "unexpected exception"));
-          }
+  // Called only from remove(), already within its cache_error_async() (op counted,
+  // existing_error() checked); see insert_raw_with_query() for why we do not re-wrap.
+  auto self = shared_from_this();
+  const couchbase::transactions::transaction_query_options opts;
+  return self->wrap_query(
+    KV_REMOVE,
+    opts,
+    make_params(document.id(), {}),
+    make_kv_txdata(document),
+    STAGE_QUERY_KV_REMOVE,
+    true,
+    {},
+    [self, id = document.id(), cb = std::move(cb)](
+      const std::exception_ptr& err, const core::operations::query_response& /* resp */) mutable {
+      if (err) {
+        try {
+          std::rethrow_exception(err);
+        } catch (const transaction_operation_failed& e) {
+          return self->op_completed_with_error(std::move(cb), e);
+        } catch (const document_not_found& e) {
+          return self->op_completed_with_error(
+            std::move(cb), transaction_operation_failed(FAIL_DOC_NOT_FOUND, e.what()).retry());
+        } catch (const query_cas_mismatch& e) {
+          return self->op_completed_with_error(
+            std::move(cb), transaction_operation_failed(FAIL_CAS_MISMATCH, e.what()).retry());
+        } catch (const op_exception& ex) {
+          return self->op_completed_with_error(std::move(cb), ex);
+        } catch (const std::exception& e) {
+          return self->op_completed_with_error(std::move(cb),
+                                               transaction_operation_failed(FAIL_OTHER, e.what()));
+        } catch (...) {
+          return self->op_completed_with_error(
+            std::move(cb), transaction_operation_failed(FAIL_OTHER, "unexpected exception"));
         }
-        // make a transaction_get_result from the row...
-        return self->op_completed_with_callback(std::move(cb));
-      });
-  });
+      }
+      // make a transaction_get_result from the row...
+      return self->op_completed_with_callback(std::move(cb));
+    });
 }
 
 void
@@ -3176,10 +3239,10 @@ attempt_context_impl::set_atr_pending_locked(
           // This bounds the value to [0-timeout].  It should always be in this
           // range, this is just to protect against the application clock
           // changing.
-          const long remaining_bounded_nanos =
+          const auto remaining_bounded_nanos =
             std::max(std::min(remaining.count(), self->overall()->config().timeout.count()),
                      static_cast<std::chrono::nanoseconds::rep>(0));
-          const long remaining_bounded_msecs = remaining_bounded_nanos / 1'000'000;
+          const auto remaining_bounded_msecs = remaining_bounded_nanos / 1'000'000;
 
           core::operations::mutate_in_request req{ self->atr_id_.value() };
 
@@ -3338,6 +3401,7 @@ attempt_context_impl::do_get(const core::document_id& id,
         return self->get_doc(
           id,
           allow_replica,
+          std::nullopt,
           [self,
            id,
            allow_replica,
@@ -3513,6 +3577,7 @@ execute_lookup(attempt_context_impl* ctx, LookupInRequest& req, Callback&& cb)
 void
 attempt_context_impl::get_doc(const core::document_id& id,
                               bool allow_replica,
+                              std::optional<std::chrono::milliseconds> timeout,
                               std::function<void(std::optional<error_class>,
                                                  std::optional<external_exception>,
                                                  std::optional<std::string>,
@@ -3536,27 +3601,21 @@ attempt_context_impl::get_doc(const core::document_id& id,
 
   try {
     if (allow_replica) {
-      cluster_ref().with_bucket_configuration(
-        id.bucket(),
-        [this, id, specs, cb = std::move(cb)](
-          std::error_code ec, const std::shared_ptr<topology::configuration>& config) {
-          if (ec) {
-            cb(FAIL_OTHER,
-               std::nullopt,
-               "failed to check whether access_deleted is supported: " + ec.message(),
-               std::nullopt);
-            return;
-          }
-          core::operations::lookup_in_any_replica_request req{ id };
-          req.read_preference = couchbase::read_preference::selected_server_group_or_all_available;
-          req.specs = specs;
-          req.access_deleted = config->capabilities.supports_subdoc_access_deleted();
-          execute_lookup(this, req, cb);
-        });
+      // Per the transactions spec ("Fetching an individual document") and MB-63241, a replica read
+      // must not be combined with the accessDeleted flag; the reference SDKs omit it on the replica
+      // path. We therefore always read replicas without accessDeleted (so tombstones are not
+      // visible there) and need no bucket-config capability probe.
+      core::operations::lookup_in_any_replica_request req{ id };
+      req.read_preference = couchbase::read_preference::selected_server_group_or_all_available;
+      req.specs = specs;
+      req.access_deleted = false;
+      req.timeout = timeout;
+      execute_lookup(this, req, cb);
     } else {
       core::operations::lookup_in_request req{ id };
       req.access_deleted = true;
       req.specs = specs;
+      req.timeout = timeout;
       execute_lookup(this, req, cb);
     }
   } catch (const std::exception& e) {
@@ -3593,8 +3652,8 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
         transaction_operation_failed(ec, "transient error in insert").retry());
     case FAIL_AMBIGUOUS:
       CB_ATTEMPT_CTX_LOG_DEBUG(this, "FAIL_AMBIGUOUS in create_staged_insert, retrying");
-      delay();
-      return create_staged_insert(id, content, cas, delay, op_id, std::forward<Handler>(cb));
+      return retry_create_staged_insert(
+        id, content, cas, std::forward<Delay>(delay), op_id, std::forward<Handler>(cb));
     case FAIL_OTHER:
       return op_completed_with_error(
         std::forward<Handler>(cb),
@@ -3655,6 +3714,7 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
           return self->get_doc(
             id,
             false,
+            std::nullopt,
             [self, id, content, op_id, cb = std::forward<Handler>(cb), error_handler, delay](
               std::optional<error_class> ec3,
               std::optional<external_exception> /* cause */,
@@ -3682,8 +3742,7 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
                                              "create staged insert found existing deleted doc, "
                                              "retrying with cas {}",
                                              doc->cas().value());
-                    delay();
-                    return self->create_staged_insert(
+                    return self->retry_create_staged_insert(
                       id, content, doc->cas().value(), delay, op_id, std::forward<Handler>(cb));
                   }
                   if (!doc->links().is_document_in_transaction()) {
@@ -3735,9 +3794,8 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
                                                "doc ok to overwrite, retrying create_staged_insert "
                                                "with cas {}",
                                                doc->cas().value());
-                      delay();
-                      return self->create_staged_insert(
-                        id, content, doc->cas().value(), delay, op_id, std::forward<Handler>(cb));
+                      return self->retry_create_staged_insert(
+                        id, content, doc->cas().value(), delay, op_id, std::move(cb));
                     });
                 } else {
                   // no doc now, just retry entire txn
@@ -3763,6 +3821,58 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
   }
 }
 
+auto
+attempt_context_impl::staged_insert_delay() const -> std::shared_ptr<async_timed_exp_delay>
+{
+  auto delay = std::make_shared<async_timed_exp_delay>(
+    std::make_shared<asio::steady_timer>(cluster_ref().io_context()),
+    std::chrono::milliseconds(5),
+    std::chrono::milliseconds(300),
+    overall()->config().timeout);
+  // The transaction expiry bounds the retries; a retry limit would fail a long transaction early.
+  delay->max_retries = std::numeric_limits<std::size_t>::max();
+  return delay;
+}
+
+template<typename Handler, typename Delay>
+void
+attempt_context_impl::retry_create_staged_insert(const core::document_id& id,
+                                                 const codec::encoded_value& content,
+                                                 std::uint64_t cas,
+                                                 Delay&& delay,
+                                                 const std::string& op_id,
+                                                 Handler&& cb)
+{
+  auto shared_delay = delay;
+  (*shared_delay)([self = shared_from_this(),
+                   id,
+                   content = content,
+                   cas,
+                   delay = std::forward<Delay>(delay),
+                   op_id,
+                   cb = std::forward<Handler>(cb)](const std::exception_ptr& delay_err) mutable {
+    if (delay_err) {
+      try {
+        std::rethrow_exception(delay_err);
+      } catch (const retry_operation_timeout&) {
+        return self->create_staged_insert_error_handler(id,
+                                                        content,
+                                                        cas,
+                                                        delay,
+                                                        op_id,
+                                                        std::move(cb),
+                                                        FAIL_EXPIRY,
+                                                        UNKNOWN,
+                                                        "create_staged_insert retries timed out");
+      } catch (const std::exception& e) {
+        return self->op_completed_with_error(std::move(cb),
+                                             transaction_operation_failed(FAIL_OTHER, e.what()));
+      }
+    }
+    return self->create_staged_insert(id, std::move(content), cas, delay, op_id, std::move(cb));
+  });
+}
+
 template<typename Handler, typename Delay>
 void
 attempt_context_impl::create_staged_insert(const core::document_id& id,
@@ -3798,6 +3908,17 @@ attempt_context_impl::create_staged_insert(const core::document_id& id,
                                               UNKNOWN,
                                               "before_staged_insert hook threw error");
   }
+  // ExtThreadSafety invariant (couchbase-transactions-specs/transactions-extensions.md,
+  // "ExtThreadSafety"): "no KV operations are staged but not sent in the BEGIN WORK" and
+  // "no KV operations are performed regularly after queryMode is entered".  insert()
+  // reads the mode only after this op is counted in the wait-group, and set_query_mode()
+  // drains all in-flight KV ops before issuing BEGIN WORK -- so reaching this staging
+  // point with query mode already entered means that ordering has regressed and the
+  // mutation would be silently lost at commit.  Fires loudly in debug/FIT builds;
+  // compiled out in release.
+  assert(!op_list_.query_mode_entered() &&
+         "ExtThreadSafety: KV insert staged after query mode entered -- would be missing "
+         "from BEGIN WORK and lost at commit");
   CB_ATTEMPT_CTX_LOG_DEBUG(this, "about to insert staged doc {} with cas {}", id, cas);
   core::operations::mutate_in_request req{ id };
   const bool binary =