couchbase 4.2.0 → 4.2.2

package/deps/couchbase-cxx-client/core/transactions/attempt_context_impl.cxx

@@ -41,6 +41,9 @@ static const std::string KV_REPLACE{ "EXECUTE __update" };
 static const std::string KV_REMOVE{ "EXECUTE __delete" };
 static const tao::json::value KV_TXDATA{ { "kv", true } };
 
+// the config may have nullptr for attempt context hooks, so we use the noop here in that case
+static auto noop_hooks = attempt_context_testing_hooks{};
+
 std::shared_ptr<core::cluster>
 attempt_context_impl::cluster_ref()
 {
@@ -50,18 +53,18 @@ attempt_context_impl::cluster_ref()
 attempt_context_impl::attempt_context_impl(transaction_context& transaction_ctx)
   : overall_(transaction_ctx)
   , staged_mutations_(std::make_unique<staged_mutation_queue>())
-  , hooks_(overall_.config().attempt_context_hooks())
+  , hooks_(overall_.config().attempt_context_hooks ? *overall_.config().attempt_context_hooks : noop_hooks)
 {
     // put a new transaction_attempt in the context...
     overall_.add_attempt();
-    trace("added new attempt, state {}, expiration in {}ms",
-          attempt_state_name(state()),
-          std::chrono::duration_cast<std::chrono::milliseconds>(overall_.remaining()).count());
+    CB_ATTEMPT_CTX_LOG_TRACE(this,
+                             "added new attempt, state {}, expiration in {}ms",
+                             attempt_state_name(state()),
+                             std::chrono::duration_cast<std::chrono::milliseconds>(overall_.remaining()).count());
 }
 
 attempt_context_impl::~attempt_context_impl() = default;
 
-// not a member of attempt_context_impl, as forward_compat is internal.
 template<typename Handler>
 void
 attempt_context_impl::check_and_handle_blocking_transactions(const transaction_get_result& doc, forward_compat_stage stage, Handler&& cb)
@@ -71,22 +74,23 @@ attempt_context_impl::check_and_handle_blocking_transactions(const transaction_g
         // Check not just writing the same doc twice in the same transaction
         // NOTE: we check the transaction rather than attempt id. This is to handle [RETRY-ERR-AMBIG-REPLACE].
         if (doc.links().staged_transaction_id().value() == transaction_id()) {
-            debug("doc {} has been written by this transaction, ok to continue", doc.id());
+            CB_ATTEMPT_CTX_LOG_DEBUG(this, "doc {} has been written by this transaction, ok to continue", doc.id());
             return cb(std::nullopt);
         }
         if (doc.links().atr_id() && doc.links().atr_bucket_name() && doc.links().staged_attempt_id()) {
-            debug("doc {} in another txn, checking atr...", doc.id());
+            CB_ATTEMPT_CTX_LOG_DEBUG(this, "doc {} in another txn, checking atr...", doc.id());
             auto err = forward_compat::check(stage, doc.links().forward_compat());
             if (err) {
                 return cb(err);
             }
             exp_delay delay(std::chrono::milliseconds(50), std::chrono::milliseconds(500), std::chrono::seconds(1));
-            return check_atr_entry_for_blocking_document(doc, delay, cb);
+            return check_atr_entry_for_blocking_document(doc, delay, std::move(cb));
         }
-        debug("doc {} is in another transaction {}, but doesn't have enough info to check the atr. "
-              "probably a bug, proceeding to overwrite",
-              doc.id(),
-              *doc.links().staged_attempt_id());
+        CB_ATTEMPT_CTX_LOG_DEBUG(this,
+                                 "doc {} is in another transaction {}, but doesn't have enough info to check the atr. "
+                                 "probably a bug, proceeding to overwrite",
+                                 doc.id(),
+                                 *doc.links().staged_attempt_id());
     }
     return cb(std::nullopt);
 }
@@ -111,7 +115,7 @@ attempt_context_impl::get(const core::document_id& id, Callback&& cb)
     if (op_list_.get_mode().is_query()) {
         return get_with_query(id, false, std::move(cb));
     }
-    cache_error_async(std::move(cb), [&]() mutable {
+    cache_error_async(cb, [&]() mutable {
         check_if_done(cb);
         do_get(
           id,
@@ -176,54 +180,62 @@ attempt_context_impl::get_optional(const core::document_id& id)
 void
 attempt_context_impl::get_optional(const core::document_id& id, Callback&& cb)
 {
+
     if (op_list_.get_mode().is_query()) {
         return get_with_query(id, true, std::move(cb));
     }
-    cache_error_async(std::move(cb), [&]() {
-        check_if_done(cb);
-        do_get(
-          id,
-          std::nullopt,
-          [this, id, cb = std::move(cb)](
-            std::optional<error_class> ec, std::optional<std::string> err_message, std::optional<transaction_get_result> res) mutable {
-              if (!ec) {
-                  ec = hooks_.after_get_complete(this, id.key());
-              }
-              if (ec) {
-                  switch (*ec) {
-                      case FAIL_EXPIRY:
-                          return op_completed_with_error(
-                            std::move(cb),
-                            transaction_operation_failed(*ec, fmt::format("transaction expired during get {}", err_message.value_or("")))
-                              .expired());
-                      case FAIL_DOC_NOT_FOUND:
-                          return op_completed_with_callback(std::move(cb), std::optional<transaction_get_result>());
-                      case FAIL_TRANSIENT:
-                          return op_completed_with_error(
-                            std::move(cb),
-                            transaction_operation_failed(*ec, fmt::format("transient failure in get {}", err_message.value_or("")))
-                              .retry());
-                      case FAIL_HARD:
-                          return op_completed_with_error(
-                            std::move(cb),
-                            transaction_operation_failed(*ec, fmt::format("fail hard in get {}", err_message.value_or(""))).no_rollback());
-                      default: {
-                          return op_completed_with_error(
-                            std::move(cb),
-                            transaction_operation_failed(FAIL_OTHER,
-                                                         fmt::format("error getting {} {}", id.key(), err_message.value_or(""))));
-                      }
-                  }
-              } else {
-                  if (res) {
-                      auto err = forward_compat::check(forward_compat_stage::GETS, res->links().forward_compat());
-                      if (err) {
-                          return op_completed_with_error(std::move(cb), *err);
-                      }
-                  }
-                  return op_completed_with_callback(std::move(cb), res);
-              }
-          });
+    cache_error_async(cb, [&]() {
+        ensure_open_bucket(id.bucket(), [this, id, cb = std::move(cb)](std::error_code ec) mutable {
+            if (ec) {
+                return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_OTHER, ec.message()));
+            }
+            check_if_done(cb);
+            do_get(id,
+                   std::nullopt,
+                   [this, id, cb = std::move(cb)](std::optional<error_class> ec,
+                                                  std::optional<std::string> err_message,
+                                                  std::optional<transaction_get_result> res) mutable {
+                       if (!ec) {
+                           ec = hooks_.after_get_complete(this, id.key());
+                       }
+                       if (ec) {
+                           switch (*ec) {
+                               case FAIL_EXPIRY:
+                                   return op_completed_with_error(
+                                     std::move(cb),
+                                     transaction_operation_failed(
+                                       *ec, fmt::format("transaction expired during get {}", err_message.value_or("")))
+                                       .expired());
+                               case FAIL_DOC_NOT_FOUND:
+                                   return op_completed_with_callback(std::move(cb), std::optional<transaction_get_result>());
+                               case FAIL_TRANSIENT:
+                                   return op_completed_with_error(
+                                     std::move(cb),
+                                     transaction_operation_failed(*ec, fmt::format("transient failure in get {}", err_message.value_or("")))
+                                       .retry());
+                               case FAIL_HARD:
+                                   return op_completed_with_error(
+                                     std::move(cb),
+                                     transaction_operation_failed(*ec, fmt::format("fail hard in get {}", err_message.value_or("")))
+                                       .no_rollback());
+                               default: {
+                                   return op_completed_with_error(
+                                     std::move(cb),
+                                     transaction_operation_failed(FAIL_OTHER,
+                                                                  fmt::format("error getting {} {}", id.key(), err_message.value_or(""))));
+                               }
+                           }
+                       } else {
+                           if (res) {
+                               auto err = forward_compat::check(forward_compat_stage::GETS, res->links().forward_compat());
+                               if (err) {
+                                   return op_completed_with_error(std::move(cb), *err);
+                               }
+                           }
+                           return op_completed_with_callback(std::move(cb), res);
+                       }
+                   });
+        });
     });
 }
 
@@ -274,62 +286,76 @@ attempt_context_impl::create_staging_request(const core::document_id& id,
 void
 attempt_context_impl::replace_raw(const transaction_get_result& document, const std::vector<std::byte>& content, Callback&& cb)
 {
+
     if (op_list_.get_mode().is_query()) {
         return replace_raw_with_query(document, content, std::move(cb));
     }
-    return cache_error_async(std::move(cb), [&]() {
-        try {
-            trace("replacing {} with {}", document, to_string(content));
-            check_if_done(cb);
-            staged_mutation* existing_sm = staged_mutations_->find_any(document.id());
-            if (existing_sm != nullptr && existing_sm->type() == staged_mutation_type::REMOVE) {
-                debug("found existing REMOVE of {} while replacing", document);
-                return op_completed_with_error(
-                  std::move(cb),
-                  transaction_operation_failed(FAIL_DOC_NOT_FOUND,
-                                               "cannot replace a document that has been removed in the same transaction")
-                    .cause(external_exception::DOCUMENT_NOT_FOUND_EXCEPTION));
-            }
-            if (check_expiry_pre_commit(STAGE_REPLACE, document.id().key())) {
-                return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_EXPIRY, "transaction expired").expired());
+    return cache_error_async(cb, [&]() {
+        ensure_open_bucket(document.bucket(), [this, cb = std::move(cb), document, content](std::error_code ec) mutable {
+            if (ec) {
+                return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_OTHER, ec.message()));
             }
-            check_and_handle_blocking_transactions(
-              document,
-              forward_compat_stage::WWC_REPLACING,
-              [this, existing_sm = std::move(existing_sm), document = std::move(document), cb = std::move(cb), content](
-                std::optional<transaction_operation_failed> e1) mutable {
-                  if (e1) {
-                      return op_completed_with_error(std::move(cb), *e1);
-                  }
-                  auto tmp_doc =
-                    document_id{ document.id().bucket(), document.id().scope(), document.id().collection(), document.id().key() };
-                  select_atr_if_needed_unlocked(
-                    tmp_doc,
-                    [this, existing_sm = std::move(existing_sm), document = std::move(document), cb = std::move(cb), content](
-                      std::optional<transaction_operation_failed> e2) mutable {
-                        if (e2) {
-                            return op_completed_with_error(std::move(cb), *e2);
-                        }
-                        if (existing_sm != nullptr && existing_sm->type() == staged_mutation_type::INSERT) {
-                            debug("found existing INSERT of {} while replacing", document);
-                            exp_delay delay(
-                              std::chrono::milliseconds(5), std::chrono::milliseconds(300), overall_.config().expiration_time());
-                            create_staged_insert(document.id(), content, existing_sm->doc().cas().value(), delay, std::move(cb));
-                            return;
-                        }
-                        create_staged_replace(document, content, std::move(cb));
-                    });
-              });
-        } catch (const client_error& e) {
-            error_class ec = e.ec();
-            switch (ec) {
-                case FAIL_EXPIRY:
-                    expiry_overtime_mode_ = true;
-                    throw transaction_operation_failed(ec, e.what()).expired();
-                default:
-                    throw transaction_operation_failed(ec, e.what());
+            try {
+                // a get can return a 'empty' doc, so check for that and short-circuit the eventual error that will occur...
+                if (document.key().empty() || document.bucket().empty()) {
+                    return op_completed_with_error(std::move(cb),
+                                                   transaction_operation_failed(FAIL_DOC_NOT_FOUND, "can't replace empty doc")
+                                                     .cause(external_exception::DOCUMENT_NOT_FOUND_EXCEPTION));
+                }
+                CB_ATTEMPT_CTX_LOG_TRACE(this, "replacing {} with {}", document, to_string(content));
+                check_if_done(cb);
+                staged_mutation* existing_sm = staged_mutations_->find_any(document.id());
+                if (existing_sm != nullptr && existing_sm->type() == staged_mutation_type::REMOVE) {
+                    CB_ATTEMPT_CTX_LOG_DEBUG(this, "found existing REMOVE of {} while replacing", document);
+                    return op_completed_with_error(
+                      std::move(cb),
+                      transaction_operation_failed(FAIL_DOC_NOT_FOUND,
+                                                   "cannot replace a document that has been removed in the same transaction")
+                        .cause(external_exception::DOCUMENT_NOT_FOUND_EXCEPTION));
+                }
+                if (check_expiry_pre_commit(STAGE_REPLACE, document.id().key())) {
+                    return op_completed_with_error(std::move(cb),
+                                                   transaction_operation_failed(FAIL_EXPIRY, "transaction expired").expired());
+                }
+
+                check_and_handle_blocking_transactions(
+                  document,
+                  forward_compat_stage::WWC_REPLACING,
+                  [this, existing_sm = std::move(existing_sm), document = std::move(document), cb = std::move(cb), content](
+                    std::optional<transaction_operation_failed> e1) mutable {
+                      if (e1) {
+                          return op_completed_with_error(std::move(cb), *e1);
+                      }
+                      auto tmp_doc =
+                        document_id{ document.id().bucket(), document.id().scope(), document.id().collection(), document.id().key() };
+                      select_atr_if_needed_unlocked(
+                        tmp_doc,
+                        [this, existing_sm = std::move(existing_sm), document = std::move(document), cb = std::move(cb), content](
+                          std::optional<transaction_operation_failed> e2) mutable {
+                            if (e2) {
+                                return op_completed_with_error(std::move(cb), *e2);
+                            }
+                            if (existing_sm != nullptr && existing_sm->type() == staged_mutation_type::INSERT) {
+                                CB_ATTEMPT_CTX_LOG_DEBUG(this, "found existing INSERT of {} while replacing", document);
+                                exp_delay delay(
+                                  std::chrono::milliseconds(5), std::chrono::milliseconds(300), overall_.config().expiration_time);
+                                create_staged_insert(document.id(), content, existing_sm->doc().cas().value(), delay, std::move(cb));
+                                return;
+                            }
+                            create_staged_replace(document, content, std::move(cb));
+                        });
+                  });
+            } catch (const client_error& e) {
+                error_class errc = e.ec();
+                switch (errc) {
+                    case FAIL_EXPIRY:
+                        expiry_overtime_mode_ = true;
+                        throw transaction_operation_failed(errc, e.what()).expired();
+                    default:
+                        throw transaction_operation_failed(errc, e.what());
+                }
             }
-        }
+        });
     });
 }
 
@@ -359,7 +385,8 @@ attempt_context_impl::create_staged_replace(const transaction_get_result& docume
     if (ec) {
         return error_handler(*ec, "before_staged_replace hook raised error", std::move(cb));
     }
-    trace("about to replace doc {} with cas {} in txn {}", document.id(), document.cas().value(), overall_.transaction_id());
+    CB_ATTEMPT_CTX_LOG_TRACE(
+      this, "about to replace doc {} with cas {} in txn {}", document.id(), document.cas().value(), overall_.transaction_id());
     overall_.cluster_ref()->execute(
       req,
       [this, document = std::move(document), content, cb = std::move(cb), error_handler = std::move(error_handler)](
@@ -374,7 +401,7 @@ attempt_context_impl::create_staged_replace(const transaction_get_result& docume
           transaction_get_result out = document;
           out.cas(resp.cas.value());
           out.content(content);
-          trace("replace staged content, result {}", out);
+          CB_ATTEMPT_CTX_LOG_TRACE(this, "replace staged content, result {}", out);
           staged_mutations_->add(staged_mutation(out, content, staged_mutation_type::REPLACE));
           return op_completed_with_callback(std::move(cb), std::optional<transaction_get_result>(out));
       });
@@ -411,68 +438,77 @@ attempt_context_impl::insert_raw(const core::document_id& id, const std::vector<
 void
 attempt_context_impl::insert_raw(const core::document_id& id, const std::vector<std::byte>& content, Callback&& cb)
 {
+
     if (op_list_.get_mode().is_query()) {
         return insert_raw_with_query(id, content, std::move(cb));
     }
-    return cache_error_async(std::move(cb), [&]() mutable {
-        try {
-            check_if_done(cb);
-            staged_mutation* existing_sm = staged_mutations_->find_any(id);
-            if ((existing_sm != nullptr) &&
-                (existing_sm->type() == staged_mutation_type::INSERT || existing_sm->type() == staged_mutation_type::REPLACE)) {
-                debug("found existing insert or replace of {} while inserting", id);
-                return op_completed_with_error(
-                  std::move(cb),
-                  transaction_operation_failed(FAIL_DOC_ALREADY_EXISTS, "found existing insert or replace of same document"));
+    return cache_error_async(cb, [&]() mutable {
+        ensure_open_bucket(id.bucket(), [this, id, content, cb = std::move(cb)](std::error_code ec) mutable {
+            if (ec) {
+                return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_OTHER, ec.message()));
             }
-            if (check_expiry_pre_commit(STAGE_INSERT, id.key())) {
-                return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_EXPIRY, "transaction expired").expired());
+            try {
+                check_if_done(cb);
+                staged_mutation* existing_sm = staged_mutations_->find_any(id);
+                if ((existing_sm != nullptr) &&
+                    (existing_sm->type() == staged_mutation_type::INSERT || existing_sm->type() == staged_mutation_type::REPLACE)) {
+                    CB_ATTEMPT_CTX_LOG_DEBUG(this, "found existing insert or replace of {} while inserting", id);
+                    return op_completed_with_error(
+                      std::move(cb),
+                      transaction_operation_failed(FAIL_DOC_ALREADY_EXISTS, "found existing insert or replace of same document"));
+                }
+                if (check_expiry_pre_commit(STAGE_INSERT, id.key())) {
+                    return op_completed_with_error(std::move(cb),
+                                                   transaction_operation_failed(FAIL_EXPIRY, "transaction expired").expired());
+                }
+                select_atr_if_needed_unlocked(
+                  id, [this, existing_sm, cb = std::move(cb), id, content](std::optional<transaction_operation_failed> err) mutable {
+                      if (err) {
+                          return op_completed_with_error(std::move(cb), *err);
+                      }
+                      if (existing_sm != nullptr && existing_sm->type() == staged_mutation_type::REMOVE) {
+                          CB_ATTEMPT_CTX_LOG_DEBUG(this, "found existing remove of {} while inserting", id);
+                          return create_staged_replace(existing_sm->doc(), content, std::move(cb));
+                      }
+                      uint64_t cas = 0;
+                      exp_delay delay(std::chrono::milliseconds(5), std::chrono::milliseconds(300), overall_.config().expiration_time);
+                      create_staged_insert(id, content, cas, delay, std::move(cb));
+                  });
+            } catch (const std::exception& e) {
+                return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_OTHER, e.what()));
             }
-            select_atr_if_needed_unlocked(
-              id, [this, existing_sm, cb = std::move(cb), id, content](std::optional<transaction_operation_failed> err) mutable {
-                  if (err) {
-                      return op_completed_with_error(std::move(cb), *err);
-                  }
-                  if (existing_sm != nullptr && existing_sm->type() == staged_mutation_type::REMOVE) {
-                      debug("found existing remove of {} while inserting", id);
-                      return create_staged_replace(existing_sm->doc(), content, std::move(cb));
-                  }
-                  uint64_t cas = 0;
-                  exp_delay delay(std::chrono::milliseconds(5), std::chrono::milliseconds(300), overall_.config().expiration_time());
-                  create_staged_insert(id, content, cas, delay, std::move(cb));
-              });
-        } catch (const std::exception& e) {
-            return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_OTHER, e.what()));
-        }
+        });
     });
 }
 
 void
 attempt_context_impl::select_atr_if_needed_unlocked(const core::document_id id,
-                                                    utils::movable_function<void(std::optional<transaction_operation_failed>)>&& cb)
+                                                    std::function<void(std::optional<transaction_operation_failed>)>&& cb)
 {
     try {
         std::unique_lock<std::mutex> lock(mutex_);
         if (atr_id_) {
-            trace("atr exists, moving on");
+            CB_ATTEMPT_CTX_LOG_TRACE(this, "atr exists, moving on");
             return cb(std::nullopt);
         }
         std::size_t vbucket_id = 0;
         std::optional<const std::string> hook_atr = hooks_.random_atr_id_for_vbucket(this);
         if (hook_atr) {
-            atr_id_ = overall_.config().atr_id_from_bucket_and_key(id.bucket(), hook_atr.value());
+            atr_id_ = atr_id_from_bucket_and_key(overall_.config(), id.bucket(), hook_atr.value());
         } else {
             vbucket_id = atr_ids::vbucket_for_key(id.key());
-            atr_id_ = overall_.config().atr_id_from_bucket_and_key(id.bucket(), atr_ids::atr_id_for_vbucket(vbucket_id));
+            atr_id_ = atr_id_from_bucket_and_key(overall_.config(), id.bucket(), atr_ids::atr_id_for_vbucket(vbucket_id));
         }
         // TODO: cleanup the transaction_context - this should be set (threadsafe) from the above calls
         overall_.atr_collection(collection_spec_from_id(id));
         overall_.atr_id(atr_id_->key());
         state(attempt_state::NOT_STARTED);
-        trace(R"(first mutated doc in transaction is "{}" on vbucket {}, so using atr "{}")", id, vbucket_id, atr_id_.value());
+        CB_ATTEMPT_CTX_LOG_TRACE(
+          this, R"(first mutated doc in transaction is "{}" on vbucket {}, so using atr "{}")", id, vbucket_id, atr_id_.value());
+        overall_.cleanup().add_collection({ atr_id_->bucket(), atr_id_->scope(), atr_id_->collection() });
         set_atr_pending_locked(id, std::move(lock), std::move(cb));
     } catch (const std::exception& e) {
-        error("unexpected error \"{}\" during select atr if needed", e.what());
+        CB_ATTEMPT_CTX_LOG_ERROR(this, "unexpected error \"{}\" during select atr if needed", e.what());
     }
 }
 template<typename Handler, typename Delay>
@@ -494,26 +530,30 @@ attempt_context_impl::check_atr_entry_for_blocking_document(const transaction_ge
           [this, delay = std::move(delay), cb = std::move(cb), doc = std::move(doc)](std::error_code err,
                                                                                      std::optional<active_transaction_record> atr) mutable {
               if (!err) {
-                  auto entries = atr->entries();
-                  auto it = std::find_if(entries.begin(), entries.end(), [&doc](const atr_entry& e) {
-                      return e.attempt_id() == doc.links().staged_attempt_id();
-                  });
-                  if (it != entries.end()) {
-                      auto fwd_err = forward_compat::check(forward_compat_stage::WWC_READING_ATR, it->forward_compat());
-                      if (fwd_err) {
-                          return cb(fwd_err);
-                      }
-                      switch (it->state()) {
-                          case attempt_state::COMPLETED:
-                          case attempt_state::ROLLED_BACK:
-                              debug("existing atr entry can be ignored due to state {}", attempt_state_name(it->state()));
-                              return cb(std::nullopt);
-                          default:
-                              debug("existing atr entry found in state {}, retrying", attempt_state_name(it->state()));
+                  if (atr) {
+                      auto entries = atr->entries();
+                      auto it = std::find_if(entries.begin(), entries.end(), [&doc](const atr_entry& e) {
+                          return e.attempt_id() == doc.links().staged_attempt_id();
+                      });
+                      if (it != entries.end()) {
+                          auto fwd_err = forward_compat::check(forward_compat_stage::WWC_READING_ATR, it->forward_compat());
+                          if (fwd_err) {
+                              return cb(fwd_err);
+                          }
+                          switch (it->state()) {
+                              case attempt_state::COMPLETED:
+                              case attempt_state::ROLLED_BACK:
+                                  CB_ATTEMPT_CTX_LOG_DEBUG(
+                                    this, "existing atr entry can be ignored due to state {}", attempt_state_name(it->state()));
+                                  return cb(std::nullopt);
+                              default:
+                                  CB_ATTEMPT_CTX_LOG_DEBUG(
+                                    this, "existing atr entry found in state {}, retrying", attempt_state_name(it->state()));
+                          }
+                          return check_atr_entry_for_blocking_document(doc, delay, std::move(cb));
                       }
-                      return check_atr_entry_for_blocking_document(doc, delay, std::move(cb));
                   }
-                  debug("no blocking atr entry");
+                  CB_ATTEMPT_CTX_LOG_DEBUG(this, "no blocking atr entry");
                   return cb(std::nullopt);
               }
               // if we are here, there is still a write-write conflict
@@ -526,90 +566,99 @@ attempt_context_impl::check_atr_entry_for_blocking_document(const transaction_ge
 void
 attempt_context_impl::remove(const transaction_get_result& document, VoidCallback&& cb)
 {
+
     if (op_list_.get_mode().is_query()) {
         return remove_with_query(document, std::move(cb));
     }
-    return cache_error_async(std::move(cb), [&]() mutable {
+    return cache_error_async(cb, [&]() mutable {
         check_if_done(cb);
-        staged_mutation* existing_sm = staged_mutations_->find_any(document.id());
-        auto error_handler = [this](error_class ec, const std::string msg, VoidCallback&& cb) mutable {
-            transaction_operation_failed err(ec, msg);
-            switch (ec) {
-                case FAIL_EXPIRY:
-                    expiry_overtime_mode_ = true;
-                    return op_completed_with_error(std::move(cb), err.expired());
-                case FAIL_DOC_NOT_FOUND:
-                case FAIL_DOC_ALREADY_EXISTS:
-                case FAIL_CAS_MISMATCH:
-                case FAIL_TRANSIENT:
-                case FAIL_AMBIGUOUS:
-                    return op_completed_with_error(std::move(cb), err.retry());
-                case FAIL_HARD:
-                    return op_completed_with_error(std::move(cb), err.no_rollback());
-                default:
-                    return op_completed_with_error(std::move(cb), err);
+        ensure_open_bucket(document.bucket(), [this, document, cb = std::move(cb)](std::error_code ec) mutable {
+            if (ec) {
+                return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_OTHER, ec.message()));
             }
-        };
-        if (check_expiry_pre_commit(STAGE_REMOVE, document.id().key())) {
-            return error_handler(FAIL_EXPIRY, "transaction expired", std::move(cb));
-        }
-        debug("removing {}", document);
-        if (existing_sm != nullptr) {
-            if (existing_sm->type() == staged_mutation_type::REMOVE) {
-                debug("found existing REMOVE of {} while removing", document);
-                return op_completed_with_error(
-                  std::move(cb),
-                  transaction_operation_failed(FAIL_DOC_NOT_FOUND, "cannot remove a document that has been removed in the same transaction")
-                    .cause(external_exception::DOCUMENT_NOT_FOUND_EXCEPTION));
+            staged_mutation* existing_sm = staged_mutations_->find_any(document.id());
+            auto error_handler = [this](error_class ec, const std::string msg, VoidCallback&& cb) mutable {
+                transaction_operation_failed err(ec, msg);
+                switch (ec) {
+                    case FAIL_EXPIRY:
+                        expiry_overtime_mode_ = true;
+                        return op_completed_with_error(std::move(cb), err.expired());
+                    case FAIL_DOC_NOT_FOUND:
+                    case FAIL_DOC_ALREADY_EXISTS:
+                    case FAIL_CAS_MISMATCH:
+                    case FAIL_TRANSIENT:
+                    case FAIL_AMBIGUOUS:
+                        return op_completed_with_error(std::move(cb), err.retry());
+                    case FAIL_HARD:
+                        return op_completed_with_error(std::move(cb), err.no_rollback());
+                    default:
+                        return op_completed_with_error(std::move(cb), err);
+                }
+            };
+            if (check_expiry_pre_commit(STAGE_REMOVE, document.id().key())) {
+                return error_handler(FAIL_EXPIRY, "transaction expired", std::move(cb));
             }
-            if (existing_sm->type() == staged_mutation_type::INSERT) {
-                remove_staged_insert(document.id(), std::move(cb));
-                return;
+            CB_ATTEMPT_CTX_LOG_DEBUG(this, "removing {}", document);
+            if (existing_sm != nullptr) {
+                if (existing_sm->type() == staged_mutation_type::REMOVE) {
+                    CB_ATTEMPT_CTX_LOG_DEBUG(this, "found existing REMOVE of {} while removing", document);
+                    return op_completed_with_error(
+                      std::move(cb),
+                      transaction_operation_failed(FAIL_DOC_NOT_FOUND,
+                                                   "cannot remove a document that has been removed in the same transaction")
+                        .cause(external_exception::DOCUMENT_NOT_FOUND_EXCEPTION));
+                }
+                if (existing_sm->type() == staged_mutation_type::INSERT) {
+                    remove_staged_insert(document.id(), std::move(cb));
+                    return;
+                }
             }
-        }
-        check_and_handle_blocking_transactions(
-          document,
-          forward_compat_stage::WWC_REMOVING,
-          [this, document = std::move(document), cb = std::move(cb), error_handler = std::move(error_handler)](
-            std::optional<transaction_operation_failed> err1) mutable {
-              if (err1) {
-                  return op_completed_with_error(std::move(cb), *err1);
-              }
-              auto tmp_doc = document_id{ document.id().bucket(), document.id().scope(), document.id().collection(), document.id().key() };
-              select_atr_if_needed_unlocked(
-                tmp_doc,
-                [document = std::move(document), cb = std::move(cb), this, error_handler = std::move(error_handler)](
-                  std::optional<transaction_operation_failed> err2) mutable {
-                    if (err2) {
-                        return op_completed_with_error(std::move(cb), *err2);
-                    }
-                    if (auto ec = hooks_.before_staged_remove(this, document.id().key())) {
-                        return error_handler(*ec, "before_staged_remove hook raised error", std::move(cb));
-                    }
-                    trace("about to remove doc {} with cas {}", document.id(), document.cas().value());
-                    auto req = create_staging_request(document.id(), &document, "remove");
-                    req.cas = document.cas();
-                    req.access_deleted = document.links().is_deleted();
-                    overall_.cluster_ref()->execute(
-                      req,
-                      [this, document = std::move(document), cb = std::move(cb), error_handler = std::move(error_handler)](
-                        core::operations::mutate_in_response resp) mutable {
-                          auto ec = error_class_from_response(resp);
-                          if (!ec) {
-                              ec = hooks_.after_staged_remove_complete(this, document.id().key());
-                          }
-                          if (!ec) {
-                              trace("removed doc {} CAS={}, rc={}", document.id(), resp.cas.value(), resp.ctx.ec().message());
-                              // TODO: this copy...  can we do better?
-                              transaction_get_result new_res = document;
-                              new_res.cas(resp.cas.value());
-                              staged_mutations_->add(staged_mutation(new_res, std::vector<std::byte>{}, staged_mutation_type::REMOVE));
-                              return op_completed_with_callback(cb);
-                          }
-                          return error_handler(*ec, resp.ctx.ec().message(), std::move(cb));
-                      });
-                });
-          });
+            check_and_handle_blocking_transactions(
+              document,
+              forward_compat_stage::WWC_REMOVING,
+              [this, document = std::move(document), cb = std::move(cb), error_handler = std::move(error_handler)](
+                std::optional<transaction_operation_failed> err1) mutable {
+                  if (err1) {
+                      return op_completed_with_error(std::move(cb), *err1);
+                  }
+                  auto tmp_doc =
+                    document_id{ document.id().bucket(), document.id().scope(), document.id().collection(), document.id().key() };
+                  select_atr_if_needed_unlocked(
+                    tmp_doc,
+                    [document = std::move(document), cb = std::move(cb), this, error_handler = std::move(error_handler)](
+                      std::optional<transaction_operation_failed> err2) mutable {
+                        if (err2) {
+                            return op_completed_with_error(std::move(cb), *err2);
+                        }
+                        if (auto ec = hooks_.before_staged_remove(this, document.id().key())) {
+                            return error_handler(*ec, "before_staged_remove hook raised error", std::move(cb));
+                        }
+                        CB_ATTEMPT_CTX_LOG_TRACE(this, "about to remove doc {} with cas {}", document.id(), document.cas().value());
+                        auto req = create_staging_request(document.id(), &document, "remove");
+                        req.cas = document.cas();
+                        req.access_deleted = document.links().is_deleted();
+                        overall_.cluster_ref()->execute(
+                          req,
+                          [this, document = std::move(document), cb = std::move(cb), error_handler = std::move(error_handler)](
+                            core::operations::mutate_in_response resp) mutable {
+                              auto ec = error_class_from_response(resp);
+                              if (!ec) {
+                                  ec = hooks_.after_staged_remove_complete(this, document.id().key());
+                              }
+                              if (!ec) {
+                                  CB_ATTEMPT_CTX_LOG_TRACE(
+                                    this, "removed doc {} CAS={}, rc={}", document.id(), resp.cas.value(), resp.ctx.ec().message());
+                                  // TODO: this copy...  can we do better?
+                                  transaction_get_result new_res = document;
+                                  new_res.cas(resp.cas.value());
+                                  staged_mutations_->add(staged_mutation(new_res, std::vector<std::byte>{}, staged_mutation_type::REMOVE));
+                                  return op_completed_with_callback(cb);
+                              }
+                              return error_handler(*ec, resp.ctx.ec().message(), std::move(cb));
+                          });
+                    });
+              });
+        });
     });
 }
 
@@ -621,7 +670,7 @@ attempt_context_impl::remove_staged_insert(const core::document_id& id, VoidCall
           std::move(cb), transaction_operation_failed(FAIL_EXPIRY, std::string("expired in remove_staged_insert")).no_rollback().expired());
     }
 
-    auto error_handler = [this, cb = std::move(cb)](error_class ec, const std::string& msg) mutable {
+    auto error_handler = [this](error_class ec, const std::string& msg, VoidCallback&& cb) mutable {
         transaction_operation_failed err(ec, msg);
         switch (ec) {
             case FAIL_HARD:
@@ -630,11 +679,10 @@ attempt_context_impl::remove_staged_insert(const core::document_id& id, VoidCall
                 return op_completed_with_error(std::move(cb), err.retry());
         }
     };
-    debug("removing staged insert {}", id);
+    CB_ATTEMPT_CTX_LOG_DEBUG(this, "removing staged insert {}", id);
 
     if (auto err = hooks_.before_remove_staged_insert(this, id.key()); err) {
-        error_handler(*err, "before_remove_staged_insert hook returned error");
-        return;
+        return error_handler(*err, "before_remove_staged_insert hook returned error", std::move(cb));
     }
 
     core::operations::mutate_in_request req{ id };
@@ -646,22 +694,23 @@ attempt_context_impl::remove_staged_insert(const core::document_id& id, VoidCall
     wrap_durable_request(req, overall_.config());
     req.access_deleted = true;
 
-    overall_.cluster_ref()->execute(
-      req, [this, id = std::move(id), cb, error_handler = std::move(error_handler)](core::operations::mutate_in_response resp) mutable {
-          auto ec = error_class_from_response(resp);
-          if (!ec) {
-              debug("remove_staged_insert got error {}", *ec);
-
-              if (auto err = hooks_.after_remove_staged_insert(this, id.key()); err) {
-                  error_handler(*err, "after_remove_staged_insert hook returned error");
-                  return;
-              }
-              staged_mutations_->remove_any(id);
-              op_completed_with_callback(std::move(cb));
-              return;
-          }
-          return error_handler(*ec, resp.ctx.ec().message());
-      });
+    overall_.cluster_ref()->execute(req,
+                                    [this, id = std::move(id), cb = std::move(cb), error_handler = std::move(error_handler)](
+                                      core::operations::mutate_in_response resp) mutable {
+                                        auto ec = error_class_from_response(resp);
+                                        if (!ec) {
+
+                                            if (auto err = hooks_.after_remove_staged_insert(this, id.key()); err) {
+                                                error_handler(*err, "after_remove_staged_insert hook returned error", std::move(cb));
+                                                return;
+                                            }
+                                            staged_mutations_->remove_any(id);
+                                            op_completed_with_callback(std::move(cb));
+                                            return;
+                                        }
+                                        CB_ATTEMPT_CTX_LOG_DEBUG(this, "remove_staged_insert got error {}", *ec);
+                                        return error_handler(*ec, resp.ctx.ec().message(), std::move(cb));
+                                    });
 }
 
 void
@@ -678,13 +727,28 @@ attempt_context_impl::remove(const transaction_get_result& document)
     f.get();
 }
 
-void
-attempt_context_impl::query_begin_work(utils::movable_function<void(std::exception_ptr)>&& cb)
+static core::operations::query_request
+wrap_query_request(const couchbase::transactions::transaction_query_options& opts, const transaction_context& txn_context)
 {
-    // check for expiry
+    // build what we can directly from the options:
+    auto req = core::impl::build_transaction_query_request(opts.get_query_options().build());
+    // set timeout to remaining time plus some extra time, so we don't time out right at expiry.
+    auto extra = core::timeout_defaults::key_value_durable_timeout;
+    if (!req.scan_consistency) {
+        req.scan_consistency = txn_context.config().query_config.scan_consistency;
+    }
+    auto remaining = std::chrono::duration_cast<std::chrono::milliseconds>(txn_context.remaining());
+    req.timeout = remaining + extra + std::chrono::milliseconds(1000); // match java with 1 second over the kv durable timeout.
+    req.raw["txtimeout"] = fmt::format("\"{}ms\"", remaining.count());
+    req.timeout = std::chrono::duration_cast<std::chrono::milliseconds>(txn_context.remaining()) + extra;
+    return req;
+}
 
+void
+attempt_context_impl::query_begin_work(std::optional<std::string> query_context, std::function<void(std::exception_ptr)>&& cb)
+{
     // construct the txn_data and query options for the existing transaction
-    transaction_query_options opts;
+    couchbase::transactions::transaction_query_options opts;
     tao::json::value txdata;
     txdata["id"] = tao::json::empty_object;
     txdata["id"]["atmpt"] = id();
@@ -693,19 +757,19 @@ attempt_context_impl::query_begin_work(utils::movable_function<void(std::excepti
     txdata["state"]["timeLeftMs"] = overall_.remaining().count() / 1000000;
     txdata["config"] = tao::json::empty_object;
     txdata["config"]["kvTimeoutMs"] =
-      overall_.config().kv_timeout() ? overall_.config().kv_timeout()->count() : core::timeout_defaults::key_value_timeout.count();
+      overall_.config().kv_timeout ? overall_.config().kv_timeout->count() : core::timeout_defaults::key_value_timeout.count();
     txdata["config"]["numAtrs"] = 1024;
     opts.raw("numatrs", jsonify(1024));
-    txdata["config"]["durabilityLevel"] = durability_level_to_string(overall_.config().durability_level());
-    opts.raw("durability_level", jsonify(durability_level_to_string_for_query(overall_.config().durability_level())));
+    txdata["config"]["durabilityLevel"] = durability_level_to_string(overall_.config().level);
+    opts.raw("durability_level", durability_level_to_string_for_query(overall_.config().level));
     if (atr_id_) {
         txdata["atr"] = tao::json::empty_object;
         txdata["atr"]["scp"] = atr_id_->scope();
         txdata["atr"]["coll"] = atr_id_->collection();
         txdata["atr"]["bkt"] = atr_id_->bucket();
         txdata["atr"]["id"] = atr_id_->key();
-    } else if (overall_.config().custom_metadata_collection()) {
-        auto id = overall_.config().atr_id_from_bucket_and_key("", "");
+    } else if (overall_.config().metadata_collection) {
+        auto id = atr_id_from_bucket_and_key(overall_.config(), "", "");
         txdata["atr"] = tao::json::empty_object;
         txdata["atr"]["scp"] = id.scope();
         txdata["atr"]["coll"] = id.collection();
@@ -727,30 +791,71 @@ attempt_context_impl::query_begin_work(utils::movable_function<void(std::excepti
     }
     txdata["mutations"] = mutations;
     std::vector<core::json_string> params;
-    trace("begin_work using txdata: {}", core::utils::json::generate(txdata));
+    CB_ATTEMPT_CTX_LOG_TRACE(this, "begin_work using txdata: {}", core::utils::json::generate(txdata));
     wrap_query(BEGIN_WORK,
                opts,
                params,
                txdata,
                STAGE_QUERY_BEGIN_WORK,
-               true,
+               false,
+               query_context,
                [this, cb = std::move(cb)](std::exception_ptr err, core::operations::query_response resp) mutable {
-                   trace("begin_work setting query node to {}", resp.served_by_node);
-                   op_list_.set_query_node(resp.served_by_node);
+                   if (resp.served_by_node.empty()) {
+                       CB_ATTEMPT_CTX_LOG_TRACE(this, "begin_work didn't reach a query node, resetting mode to kv");
+                       op_list_.reset_query_mode();
+                   } else {
+                       CB_ATTEMPT_CTX_LOG_TRACE(this, "begin_work setting query node to {}", resp.served_by_node);
+                       op_list_.set_query_node(resp.served_by_node);
+                   }
+                   // we check for expiry _after_ this call, so we always set the query node if we can.
+                   if (has_expired_client_side(STAGE_QUERY_BEGIN_WORK, {})) {
+                       return cb(std::make_exception_ptr(
+                         transaction_operation_failed(FAIL_EXPIRY, "expired in BEGIN WORK").no_rollback().expired()));
+                   }
                    return cb(err);
                });
 }
 
+tao::json::value
+choose_error(std::vector<tao::json::value>& errors)
+{
+    auto chosen_error = errors.front();
+    if (errors.size() > 1) {
+        // if there's one with a "reason":{"cause", ...} field, choose it
+        for (const auto& e : errors) {
+            auto reason = e.find("reason");
+            auto cause = e.find("cause");
+            if (reason && !reason->is_null() && cause && !cause->is_null()) {
+                return e;
+            }
+        }
+        // ok, so now lets see if we have one with code in the range 17000-18000 and return that.
+        for (const auto& e : errors) {
+            auto code = e.at("code").as<uint64_t>();
+            if (code >= 17000 && code <= 18000) {
+                return e;
+            }
+        }
+    }
+    // then, just the first one.
+    return chosen_error;
+}
+
 std::exception_ptr
 attempt_context_impl::handle_query_error(const core::operations::query_response& resp)
 {
     if (!resp.ctx.ec && !resp.meta.errors) {
         return {};
     }
+    auto tx_resp = couchbase::core::impl::build_transaction_query_result(resp);
     // TODO: look at ambiguous and unambiguous timeout errors vs the codes, etc...
-    trace("handling query error {}, {} errors in meta_data", resp.ctx.ec.message(), resp.meta.errors ? "has" : "no");
+    CB_ATTEMPT_CTX_LOG_TRACE(
+      this, "handling query error {}, {} errors in meta_data", resp.ctx.ec.message(), resp.meta.errors ? "has" : "no");
     if (resp.ctx.ec == couchbase::errc::common::ambiguous_timeout || resp.ctx.ec == couchbase::errc::common::unambiguous_timeout) {
-        return std::make_exception_ptr(query_attempt_expired(resp.ctx.ec.message()));
+        return std::make_exception_ptr(query_attempt_expired(tx_resp.ctx()));
+    }
+    if (resp.ctx.ec == couchbase::errc::common::parsing_failure) {
+        return std::make_exception_ptr(query_parsing_failure(tx_resp.ctx()));
     }
     if (!resp.meta.errors) {
         // can't choose an error, map using the ec...
@@ -758,77 +863,84 @@ attempt_context_impl::handle_query_error(const core::operations::query_response&
           (resp.ctx.ec == couchbase::errc::common::service_not_available ? SERVICE_NOT_AVAILABLE_EXCEPTION : COUCHBASE_EXCEPTION);
         return std::make_exception_ptr(transaction_operation_failed(FAIL_OTHER, resp.ctx.ec.message()).cause(cause));
     }
-    auto chosen_error = resp.meta.errors->front();
-    for (const auto& err : *resp.meta.errors) {
-        if (err.code >= 17000 && err.code <= 18000) {
-            chosen_error = err;
-            break;
-        }
-    }
-    trace("chosen query error ({}):'{}'", chosen_error.code, chosen_error.message);
-    switch (chosen_error.code) {
+    // TODO: we should have all the fields in these transactional query errors in the errors object.  For now, lets
+    //   parse the body, get the serialized info to decide which error to choose.
+    auto errors = core::utils::json::parse(resp.ctx.http_body)["errors"].get_array();
+
+    // just chose first one, to start with...
+    auto chosen_error = choose_error(errors);
+    CB_ATTEMPT_CTX_LOG_TRACE(this, "chosen query error: {}", jsonify(chosen_error));
+    auto code = chosen_error.at("code").as<uint64_t>();
+
+    // we have a fixed strategy for these errors...
+    switch (code) {
         case 1065:
             return std::make_exception_ptr(
               transaction_operation_failed(FAIL_OTHER, "N1QL Queries in transactions are supported in couchbase server 7.0 and later")
                 .cause(FEATURE_NOT_AVAILABLE_EXCEPTION));
-        case 3000:
-            return std::make_exception_ptr(query_parsing_failure(chosen_error.message));
+        case 1197:
+            return std::make_exception_ptr(
+              transaction_operation_failed(FAIL_OTHER, "This couchbase server requires all queries use a scope.")
+                .cause(FEATURE_NOT_AVAILABLE_EXCEPTION));
         case 17004:
-            return std::make_exception_ptr(query_attempt_not_found(chosen_error.message));
+            return std::make_exception_ptr(query_attempt_not_found(tx_resp.ctx()));
         case 1080:
         case 17010:
-            return std::make_exception_ptr(transaction_operation_failed(FAIL_EXPIRY, "transaction expired"));
+            return std::make_exception_ptr(transaction_operation_failed(FAIL_EXPIRY, "transaction expired").expired());
         case 17012:
-            return std::make_exception_ptr(query_document_exists(chosen_error.message));
+            return std::make_exception_ptr(document_exists(tx_resp.ctx()));
         case 17014:
-            return std::make_exception_ptr(query_document_not_found(chosen_error.message));
+            return std::make_exception_ptr(document_not_found(tx_resp.ctx()));
         case 17015:
-            return std::make_exception_ptr(query_cas_mismatch(chosen_error.message));
+            return std::make_exception_ptr(query_cas_mismatch(tx_resp.ctx()));
     }
-    if (chosen_error.code >= 17000 && chosen_error.code <= 18000) {
-        transaction_operation_failed err(FAIL_OTHER, chosen_error.message);
+
+    // For these errors, we will create a transaction_operation_failed from the info in it.
+    if (code >= 17000 && code <= 18000) {
+        // the assumption below is there's always a top-level msg.
+        // TODO: when we parse the errors more thoroughly in the client, we should be able to add a lot of info on the underlying
+        //   cause of the error here (in addition to perhaps a more granular message).
+        transaction_operation_failed err(FAIL_OTHER, chosen_error.at("msg").as<std::string>());
         // parse the body for now, get the serialized info to create a transaction_operation_failed:
-        auto body = core::utils::json::parse(resp.ctx.http_body);
-        auto errors = body["errors"];
-        for (const auto& e : errors.get_array()) {
-            if (e.at("code").as<std::uint64_t>() == chosen_error.code) {
-                if (const auto* cause = e.find("cause"); cause != nullptr) {
-                    if (cause->at("retry").get_boolean()) {
-                        err.retry();
-                    }
-                    if (!cause->at("rollback").get_boolean()) {
-                        err.no_rollback();
-                    }
-                    if (auto raise = cause->at("raise").get_string(); raise == std::string("expired")) {
-                        err.expired();
-                    } else if (raise == std::string("commit_ambiguous")) {
-                        err.ambiguous();
-                    } else if (raise == std::string("failed_post_commit")) {
-                        err.failed_post_commit();
-                    } else if (raise != std::string("failed")) {
-                        trace("unknown value in raise field: {}, raising failed", raise);
-                    }
-                    return std::make_exception_ptr(err);
-                }
+        if (const auto* cause = chosen_error.find("cause"); cause != nullptr) {
+            if (cause->find("retry")->get_boolean()) {
+                err.retry();
+            }
+            if (!cause->find("rollback")->get_boolean()) {
+                err.no_rollback();
+            }
+            if (auto raise = cause->find("raise")->get_string(); raise == std::string("expired")) {
+                err.expired();
+            } else if (raise == std::string("commit_ambiguous")) {
+                err.ambiguous();
+            } else if (raise == std::string("failed_post_commit")) {
+                err.failed_post_commit();
+            } else if (raise != std::string("failed")) {
+                CB_ATTEMPT_CTX_LOG_TRACE(this, "unknown value in raise field: {}, raising failed", raise);
             }
+            return std::make_exception_ptr(err);
         }
     }
 
-    return { std::make_exception_ptr(query_exception(chosen_error.message)) };
+    return { std::make_exception_ptr(op_exception(tx_resp.ctx())) };
 }
 
 void
-attempt_context_impl::do_query(const std::string& statement, const transaction_query_options& opts, QueryCallback&& cb)
+attempt_context_impl::do_query(const std::string& statement,
+                               const couchbase::transactions::transaction_query_options& opts,
+                               std::optional<std::string> query_context,
+                               QueryCallback&& cb)
 {
     std::vector<core::json_string> params;
     tao::json::value txdata;
-    trace("do_query called with statement {}", statement);
+    CB_ATTEMPT_CTX_LOG_TRACE(this, "do_query called with statement {}", statement);
     wrap_query(statement,
                opts,
                params,
                txdata,
                STAGE_QUERY,
                true,
+               query_context,
                [this, cb = std::move(cb)](std::exception_ptr err, core::operations::query_response resp) mutable {
                    if (err) {
                        return op_completed_with_error(std::move(cb), err);
@@ -855,14 +967,15 @@ dump_request(const core::operations::query_request& req)
 }
 void
 attempt_context_impl::wrap_query(const std::string& statement,
-                                 const transaction_query_options& opts,
+                                 const couchbase::transactions::transaction_query_options& opts,
                                  const std::vector<core::json_string>& params,
                                  const tao::json::value& txdata,
                                  const std::string& hook_point,
                                  bool check_expiry,
-                                 utils::movable_function<void(std::exception_ptr, core::operations::query_response)>&& cb)
+                                 std::optional<std::string> query_context,
+                                 std::function<void(std::exception_ptr, core::operations::query_response)>&& cb)
 {
-    auto req = opts.wrap_request(overall_);
+    auto req = wrap_query_request(opts, overall_);
     if (statement != BEGIN_WORK) {
         auto mode = op_list_.get_mode();
         assert(mode.is_query());
@@ -870,6 +983,13 @@ attempt_context_impl::wrap_query(const std::string& statement,
             req.send_to_node = op_list_.get_mode().query_node;
         }
     }
+    // set the query_context, if one has been set, unless this query already has one
+    if (!query_context && !query_context_.empty()) {
+        req.query_context = query_context_;
+    } else if (query_context) {
+        req.query_context = query_context;
+    }
+
     if (check_expiry) {
         if (has_expired_client_side(hook_point, std::nullopt)) {
             auto err = std::make_exception_ptr(
@@ -897,9 +1017,9 @@ attempt_context_impl::wrap_query(const std::string& statement,
         }
         return cb(std::make_exception_ptr(transaction_operation_failed(*ec, "before_query hook raised error")), {});
     }
-    trace("http request: {}", dump_request(req));
+    CB_ATTEMPT_CTX_LOG_TRACE(this, "http request: {}", dump_request(req));
     overall_.cluster_ref()->execute(req, [this, cb = std::move(cb)](core::operations::query_response resp) mutable {
-        trace("response: {} status: {}", resp.ctx.http_body, resp.meta.status);
+        CB_ATTEMPT_CTX_LOG_TRACE(this, "response: {} status: {}", resp.ctx.http_body, resp.meta.status);
         if (auto ec = hooks_.after_query(this, resp.ctx.statement)) {
             auto err = std::make_exception_ptr(transaction_operation_failed(*ec, "after_query hook raised error"));
             return cb(err, {});
@@ -909,30 +1029,40 @@ attempt_context_impl::wrap_query(const std::string& statement,
 }
 
 void
-attempt_context_impl::query(const std::string& statement, const transaction_query_options& options, QueryCallback&& cb)
+attempt_context_impl::query(const std::string& statement,
+                            const couchbase::transactions::transaction_query_options& options,
+                            std::optional<std::string> query_context,
+                            QueryCallback&& cb)
 {
-    return cache_error_async(std::move(cb), [&]() {
+    return cache_error_async(cb, [&]() {
         check_if_done(cb);
         // decrement in_flight, as we just incremented it in cache_error_async.
         op_list_.set_query_mode(
-          [this, statement, options, cb]() mutable {
-              query_begin_work([this, statement, options, cb = std::move(cb)](std::exception_ptr err) mutable {
-                  if (err) {
-                      return op_completed_with_error(std::move(cb), err);
-                  }
-                  return do_query(statement, options, std::move(cb));
-              });
+          [this, statement, options, query_context, cb]() mutable {
+              // set query context if set
+              if (query_context) {
+                  query_context_ = query_context.value();
+              }
+              query_begin_work(query_context,
+                               [this, statement, query_context, options, cb = std::move(cb)](std::exception_ptr err) mutable {
+                                   if (err) {
+                                       return op_completed_with_error(std::move(cb), err);
+                                   }
+                                   return do_query(statement, options, query_context, std::move(cb));
+                               });
           },
-          [this, statement = std::move(statement), options, cb]() mutable { return do_query(statement, options, std::move(cb)); });
+          [this, statement, options, query_context, cb]() mutable { return do_query(statement, options, query_context, std::move(cb)); });
     });
 }
 
 core::operations::query_response
-attempt_context_impl::query(const std::string& statement, const transaction_query_options& options)
+attempt_context_impl::do_core_query(const std::string& statement,
+                                    const couchbase::transactions::transaction_query_options& options,
+                                    std::optional<std::string> query_context)
 {
     auto barrier = std::make_shared<std::promise<core::operations::query_response>>();
     auto f = barrier->get_future();
-    query(statement, options, [barrier](std::exception_ptr err, std::optional<core::operations::query_response> resp) {
+    query(statement, options, query_context, [barrier](std::exception_ptr err, std::optional<core::operations::query_response> resp) {
         if (err) {
             return barrier->set_exception(err);
         }
@@ -941,6 +1071,25 @@ attempt_context_impl::query(const std::string& statement, const transaction_quer
     return f.get();
 }
 
+couchbase::transactions::transaction_query_result_ptr
+attempt_context_impl::do_public_query(const std::string& statement,
+                                      const couchbase::transactions::transaction_query_options& opts,
+                                      std::optional<std::string> query_context)
+{
+    try {
+        auto result = do_core_query(statement, opts, query_context);
+        return std::make_shared<couchbase::transactions::transaction_query_result>(core::impl::build_transaction_query_result(result));
+    } catch (const transaction_operation_failed& e) {
+        return std::make_shared<couchbase::transactions::transaction_query_result>(e.get_error_ctx());
+    } catch (const op_exception& qe) {
+        return std::make_shared<couchbase::transactions::transaction_query_result>(qe.ctx());
+    } catch (...) {
+        // should not be necessary, but just in case...
+        transaction_op_error_context ctx(couchbase::errc::transaction_op::unknown);
+        return std::make_shared<couchbase::transactions::transaction_query_result>(ctx);
+    }
+}
+
 std::vector<core::json_string>
 make_params(const core::document_id& id, std::optional<std::vector<std::byte>> content)
 {
@@ -971,9 +1120,9 @@ make_kv_txdata(std::optional<transaction_get_result> doc = std::nullopt)
 void
 attempt_context_impl::get_with_query(const core::document_id& id, bool optional, Callback&& cb)
 {
-    cache_error_async(std::move(cb), [&]() {
+    cache_error_async(cb, [&]() {
         auto params = make_params(id, {});
-        transaction_query_options opts;
+        couchbase::transactions::transaction_query_options opts;
         opts.readonly(true);
         return wrap_query(KV_GET,
                           opts,
@@ -981,6 +1130,7 @@ attempt_context_impl::get_with_query(const core::document_id& id, bool optional,
                           make_kv_txdata(),
                           STAGE_QUERY_KV_GET,
                           true,
+                          {},
                           [this, id, optional, cb = std::move(cb)](std::exception_ptr err, core::operations::query_response resp) mutable {
                               if (resp.ctx.ec == couchbase::errc::key_value::document_not_found) {
                                   return op_completed_with_callback(std::move(cb), std::optional<transaction_get_result>());
@@ -989,10 +1139,10 @@ attempt_context_impl::get_with_query(const core::document_id& id, bool optional,
                                   // make a transaction_get_result from the row...
                                   try {
                                       if (resp.rows.empty()) {
-                                          trace("get_with_query got no doc and no error, returning query_document_not_found");
-                                          return op_completed_with_error(std::move(cb), query_document_not_found("doc not found"));
+                                          return op_completed_with_error(
+                                            std::move(cb), transaction_operation_failed(FAIL_DOC_NOT_FOUND, "document not found"));
                                       }
-                                      trace("get_with_query got: {}", resp.rows.front());
+                                      CB_ATTEMPT_CTX_LOG_TRACE(this, "get_with_query got: {}", resp.rows.front());
                                       transaction_get_result doc(id, core::utils::json::parse(resp.rows.front()));
                                       return op_completed_with_callback(std::move(cb), std::optional<transaction_get_result>(doc));
                                   } catch (const std::exception& e) {
@@ -1004,7 +1154,7 @@ attempt_context_impl::get_with_query(const core::document_id& id, bool optional,
                               if (optional) {
                                   try {
                                       std::rethrow_exception(err);
-                                  } catch (const query_document_not_found&) {
+                                  } catch (const document_not_found&) {
                                       return op_completed_with_callback(std::move(cb), std::optional<transaction_get_result>());
                                   } catch (...) {
                                       return op_completed_with_error(std::move(cb), std::current_exception());
@@ -1018,24 +1168,26 @@ attempt_context_impl::get_with_query(const core::document_id& id, bool optional,
 void
 attempt_context_impl::insert_raw_with_query(const core::document_id& id, const std::vector<std::byte>& content, Callback&& cb)
 {
-    cache_error_async(std::move(cb), [&]() {
+    cache_error_async(cb, [&]() {
         std::vector<std::byte> content_copy = content;
         auto params = make_params(id, std::move(content_copy));
-        transaction_query_options opts;
+        couchbase::transactions::transaction_query_options opts;
         return wrap_query(KV_INSERT,
                           opts,
                           params,
                           make_kv_txdata(),
                           STAGE_QUERY_KV_INSERT,
                           true,
+                          {},
                           [this, id, cb = std::move(cb)](std::exception_ptr err, core::operations::query_response resp) mutable {
                               if (err) {
                                   try {
                                       std::rethrow_exception(err);
                                   } catch (const transaction_operation_failed&) {
                                       return op_completed_with_error(std::move(cb), err);
-                                  } catch (const query_document_exists& e) {
-                                      return op_completed_with_error(std::move(cb), e);
+
+                                  } catch (const document_exists& ex) {
+                                      return op_completed_with_error(std::move(cb), ex);
                                   } catch (const std::exception& e) {
                                       return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_OTHER, e.what()));
                                   } catch (...) {
@@ -1045,7 +1197,7 @@ attempt_context_impl::insert_raw_with_query(const core::document_id& id, const s
                               }
                               // make a transaction_get_result from the row...
                               try {
-                                  trace("insert_raw_with_query got: {}", resp.rows.front());
+                                  CB_ATTEMPT_CTX_LOG_TRACE(this, "insert_raw_with_query got: {}", resp.rows.front());
                                   transaction_get_result doc(id, core::utils::json::parse(resp.rows.front()));
                                   return op_completed_with_callback(std::move(cb), std::optional<transaction_get_result>(doc));
                               } catch (const std::exception& e) {
@@ -1059,10 +1211,10 @@ attempt_context_impl::insert_raw_with_query(const core::document_id& id, const s
 void
 attempt_context_impl::replace_raw_with_query(const transaction_get_result& document, const std::vector<std::byte>& content, Callback&& cb)
 {
-    cache_error_async(std::move(cb), [&]() {
+    cache_error_async(cb, [&]() {
         std::vector<std::byte> content_copy = content;
         auto params = make_params(document.id(), std::move(content_copy));
-        transaction_query_options opts;
+        couchbase::transactions::transaction_query_options opts;
         return wrap_query(
           KV_REPLACE,
           opts,
@@ -1070,13 +1222,14 @@ attempt_context_impl::replace_raw_with_query(const transaction_get_result& docum
           make_kv_txdata(document),
           STAGE_QUERY_KV_REPLACE,
           true,
+          {},
           [this, id = document.id(), cb = std::move(cb)](std::exception_ptr err, core::operations::query_response resp) mutable {
               if (err) {
                   try {
                       std::rethrow_exception(err);
                   } catch (const query_cas_mismatch& e) {
                       return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_CAS_MISMATCH, e.what()).retry());
-                  } catch (const query_document_not_found& e) {
+                  } catch (const document_not_found& e) {
                       return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_DOC_NOT_FOUND, e.what()).retry());
                   } catch (const transaction_operation_failed& e) {
                       return op_completed_with_error(std::move(cb), e);
@@ -1088,7 +1241,7 @@ attempt_context_impl::replace_raw_with_query(const transaction_get_result& docum
               }
               // make a transaction_get_result from the row...
               try {
-                  trace("replace_raw_with_query got: {}", resp.rows.front());
+                  CB_ATTEMPT_CTX_LOG_TRACE(this, "replace_raw_with_query got: {}", resp.rows.front());
                   transaction_get_result doc(id, core::utils::json::parse(resp.rows.front()));
                   return op_completed_with_callback(std::move(cb), std::optional<transaction_get_result>(doc));
               } catch (const std::exception& e) {
@@ -1102,9 +1255,9 @@ attempt_context_impl::replace_raw_with_query(const transaction_get_result& docum
 void
 attempt_context_impl::remove_with_query(const transaction_get_result& document, VoidCallback&& cb)
 {
-    cache_error_async(std::move(cb), [&]() {
+    cache_error_async(cb, [&]() {
         auto params = make_params(document.id(), {});
-        transaction_query_options opts;
+        couchbase::transactions::transaction_query_options opts;
         return wrap_query(
           KV_REMOVE,
           opts,
@@ -1112,13 +1265,14 @@ attempt_context_impl::remove_with_query(const transaction_get_result& document,
           make_kv_txdata(document),
           STAGE_QUERY_KV_REMOVE,
           true,
+          {},
           [this, id = document.id(), cb = std::move(cb)](std::exception_ptr err, core::operations::query_response /* resp */) mutable {
               if (err) {
                   try {
                       std::rethrow_exception(err);
                   } catch (const transaction_operation_failed& e) {
                       return op_completed_with_error(std::move(cb), e);
-                  } catch (const query_document_not_found& e) {
+                  } catch (const document_not_found& e) {
                       return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_DOC_NOT_FOUND, e.what()).retry());
                   } catch (const query_cas_mismatch& e) {
                       return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_CAS_MISMATCH, e.what()).retry());
@@ -1138,8 +1292,8 @@ void
 attempt_context_impl::commit_with_query(VoidCallback&& cb)
 {
     core::operations::query_request req;
-    trace("commit_with_query called");
-    transaction_query_options opts;
+    CB_ATTEMPT_CTX_LOG_TRACE(this, "commit_with_query called");
+    couchbase::transactions::transaction_query_options opts;
     std::vector<core::json_string> params;
     wrap_query(
       COMMIT,
@@ -1148,6 +1302,7 @@ attempt_context_impl::commit_with_query(VoidCallback&& cb)
       make_kv_txdata(std::nullopt),
       STAGE_QUERY_COMMIT,
       true,
+      {},
       [this, cb = std::move(cb)](std::exception_ptr err, core::operations::query_response /* resp */) mutable {
           is_done_ = true;
           if (err) {
@@ -1157,9 +1312,9 @@ attempt_context_impl::commit_with_query(VoidCallback&& cb)
                   return cb(std::current_exception());
               } catch (const query_attempt_expired& e) {
                   return cb(std::make_exception_ptr(transaction_operation_failed(FAIL_EXPIRY, e.what()).ambiguous().no_rollback()));
-              } catch (const query_document_not_found& e) {
+              } catch (const document_not_found& e) {
                   return cb(std::make_exception_ptr(transaction_operation_failed(FAIL_DOC_NOT_FOUND, e.what()).no_rollback()));
-              } catch (const query_document_exists& e) {
+              } catch (const document_exists& e) {
                   return cb(std::make_exception_ptr(transaction_operation_failed(FAIL_DOC_ALREADY_EXISTS, e.what()).no_rollback()));
               } catch (const query_cas_mismatch& e) {
                   return cb(std::make_exception_ptr(transaction_operation_failed(FAIL_CAS_MISMATCH, e.what()).no_rollback()));
@@ -1175,8 +1330,8 @@ void
 attempt_context_impl::rollback_with_query(VoidCallback&& cb)
 {
     core::operations::query_request req;
-    trace("rollback_with_query called");
-    transaction_query_options opts;
+    CB_ATTEMPT_CTX_LOG_TRACE(this, "rollback_with_query called");
+    couchbase::transactions::transaction_query_options opts;
     std::vector<core::json_string> params;
     wrap_query(ROLLBACK,
                opts,
@@ -1184,6 +1339,7 @@ attempt_context_impl::rollback_with_query(VoidCallback&& cb)
                make_kv_txdata(std::nullopt),
                STAGE_QUERY_ROLLBACK,
                true,
+               {},
                [this, cb = std::move(cb)](std::exception_ptr err, core::operations::query_response /* resp */) mutable {
                    is_done_ = true;
                    if (err) {
@@ -1192,13 +1348,14 @@ attempt_context_impl::rollback_with_query(VoidCallback&& cb)
                        } catch (const transaction_operation_failed&) {
                            return cb(std::current_exception());
                        } catch (const query_attempt_not_found& e) {
-                           debug("got query_attempt_not_found, assuming query was already rolled back successfullly: {}", e.what());
+                           CB_ATTEMPT_CTX_LOG_DEBUG(
+                             this, "got query_attempt_not_found, assuming query was already rolled back successfullly: {}", e.what());
                        } catch (const std::exception& e) {
                            return cb(std::make_exception_ptr(transaction_operation_failed(FAIL_OTHER, e.what()).no_rollback()));
                        }
                    }
                    state(attempt_state::ROLLED_BACK);
-                   trace("rollback successful");
+                   CB_ATTEMPT_CTX_LOG_TRACE(this, "rollback successful");
                    return cb({});
                });
 }
@@ -1228,7 +1385,7 @@ attempt_context_impl::atr_commit(bool ambiguity_resolution_mode)
             staged_mutations_->extract_to(prefix, req);
             auto barrier = std::make_shared<std::promise<result>>();
             auto f = barrier->get_future();
-            trace("updating atr {}", req.id);
+            CB_ATTEMPT_CTX_LOG_TRACE(this, "updating atr {}, setting to {}", req.id, attempt_state_name(attempt_state::COMMITTED));
             overall_.cluster_ref()->execute(
               req, [barrier](core::operations::mutate_in_response resp) { barrier->set_value(result::create_from_subdoc_response(resp)); });
             auto res = wrap_operation_future(f, false);
@@ -1251,7 +1408,7 @@ attempt_context_impl::atr_commit(bool ambiguity_resolution_mode)
                     throw out;
                 }
                 case FAIL_AMBIGUOUS:
-                    debug("atr_commit got FAIL_AMBIGUOUS, resolving ambiguity...");
+                    CB_ATTEMPT_CTX_LOG_DEBUG(this, "atr_commit got FAIL_AMBIGUOUS, resolving ambiguity...");
                     ambiguity_resolution_mode = true;
                     throw retry_operation(e.what());
                 case FAIL_TRANSIENT:
@@ -1297,11 +1454,12 @@ attempt_context_impl::atr_commit(bool ambiguity_resolution_mode)
                     throw out;
                 }
                 default: {
-                    error("failed to commit transaction {}, attempt {}, ambiguity_resolution_mode {}, with error {}",
-                          transaction_id(),
-                          id(),
-                          ambiguity_resolution_mode,
-                          e.what());
+                    CB_ATTEMPT_CTX_LOG_ERROR(this,
+                                             "failed to commit transaction {}, attempt {}, ambiguity_resolution_mode {}, with error {}",
+                                             transaction_id(),
+                                             id(),
+                                             ambiguity_resolution_mode,
+                                             e.what());
                     auto out = transaction_operation_failed(ec, e.what());
                     if (ambiguity_resolution_mode) {
                         out.no_rollback().ambiguous();
@@ -1334,7 +1492,7 @@ attempt_context_impl::atr_commit_ambiguity_resolution()
           req, [barrier](core::operations::lookup_in_response resp) { barrier->set_value(result::create_from_subdoc_response(resp)); });
         auto res = wrap_operation_future(f);
         auto atr_status_raw = res.values[0].content_as<std::string>();
-        debug("atr_commit_ambiguity_resolution read atr state {}", atr_status_raw);
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "atr_commit_ambiguity_resolution read atr state {}", atr_status_raw);
         auto atr_status = attempt_state_value(atr_status_raw);
         switch (atr_status) {
             case attempt_state::COMMITTED:
@@ -1380,7 +1538,7 @@ attempt_context_impl::atr_complete()
         if (!!(ec = error_if_expired_and_not_in_overtime(STAGE_ATR_COMPLETE, {}))) {
             throw client_error(*ec, "atr_complete threw error");
         }
-        debug("removing attempt {} from atr", atr_id_.value());
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "removing attempt {} from atr", atr_id_.value());
         std::string prefix(ATR_FIELD_ATTEMPTS + "." + id());
         core::operations::mutate_in_request req{ atr_id_.value() };
         req.specs =
@@ -1405,7 +1563,7 @@ attempt_context_impl::atr_complete()
             case FAIL_HARD:
                 throw transaction_operation_failed(ec, er.what()).no_rollback().failed_post_commit();
             default:
-                info("ignoring error in atr_complete {}", er.what());
+                CB_ATTEMPT_CTX_LOG_INFO(this, "ignoring error in atr_complete {}", er.what());
         }
     }
 }
@@ -1428,10 +1586,10 @@ attempt_context_impl::commit(VoidCallback&& cb)
 void
 attempt_context_impl::commit()
 {
-    debug("waiting on ops to finish...");
+    CB_ATTEMPT_CTX_LOG_DEBUG(this, "waiting on ops to finish...");
     op_list_.wait_and_block_ops();
     existing_error(false);
-    debug("commit {}", id());
+    CB_ATTEMPT_CTX_LOG_DEBUG(this, "commit {}", id());
     if (op_list_.get_mode().is_query()) {
         auto barrier = std::make_shared<std::promise<void>>();
         auto f = barrier->get_future();
@@ -1449,13 +1607,13 @@ attempt_context_impl::commit()
         }
         if (atr_id_ && !atr_id_->key().empty() && !is_done_) {
             retry_op_exp<void>([&]() { atr_commit(false); });
-            staged_mutations_->commit(*this);
+            staged_mutations_->commit(this);
             atr_complete();
             is_done_ = true;
         } else {
             // no mutation, no need to commit
             if (!is_done_) {
-                debug("calling commit on attempt that has got no mutations, skipping");
+                CB_ATTEMPT_CTX_LOG_DEBUG(this, "calling commit on attempt that has got no mutations, skipping");
                 is_done_ = true;
                 return;
             } // do not rollback or retry
@@ -1499,17 +1657,17 @@ attempt_context_impl::atr_abort()
         if (ec) {
             throw client_error(*ec, "after_atr_aborted hook threw error");
         }
-        debug("rollback completed atr abort phase");
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "rollback completed atr abort phase");
     } catch (const client_error& e) {
         auto ec = e.ec();
-        trace("atr_abort got {} {}", ec, e.what());
+        CB_ATTEMPT_CTX_LOG_TRACE(this, "atr_abort got {} {}", ec, e.what());
         if (expiry_overtime_mode_.load()) {
-            debug("atr_abort got error \"{}\" while in overtime mode", e.what());
+            CB_ATTEMPT_CTX_LOG_DEBUG(this, "atr_abort got error \"{}\" while in overtime mode", e.what());
             throw transaction_operation_failed(FAIL_EXPIRY, std::string("expired in atr_abort with {} ") + e.what())
               .no_rollback()
               .expired();
         }
-        debug("atr_abort got error {}", ec);
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "atr_abort got error {}", ec);
         switch (ec) {
             case FAIL_EXPIRY:
                 expiry_overtime_mode_ = true;
@@ -1562,28 +1720,28 @@ attempt_context_impl::atr_rollback_complete()
     } catch (const client_error& e) {
         auto ec = e.ec();
         if (expiry_overtime_mode_.load()) {
-            debug("atr_rollback_complete error while in overtime mode {}", e.what());
+            CB_ATTEMPT_CTX_LOG_DEBUG(this, "atr_rollback_complete error while in overtime mode {}", e.what());
             throw transaction_operation_failed(FAIL_EXPIRY, std::string("expired in atr_rollback_complete with {} ") + e.what())
               .no_rollback()
               .expired();
         }
-        debug("atr_rollback_complete got error {}", ec);
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "atr_rollback_complete got error {}", ec);
         switch (ec) {
             case FAIL_DOC_NOT_FOUND:
             case FAIL_PATH_NOT_FOUND:
-                debug("atr {} not found, ignoring", atr_id_->key());
+                CB_ATTEMPT_CTX_LOG_DEBUG(this, "atr {} not found, ignoring", atr_id_->key());
                 is_done_ = true;
                 break;
             case FAIL_ATR_FULL:
-                debug("atr {} full!", atr_id_->key());
+                CB_ATTEMPT_CTX_LOG_DEBUG(this, "atr {} full!", atr_id_->key());
                 throw retry_operation(e.what());
             case FAIL_HARD:
                 throw transaction_operation_failed(ec, e.what()).no_rollback();
             case FAIL_EXPIRY:
-                debug("timed out writing atr {}", atr_id_->key());
+                CB_ATTEMPT_CTX_LOG_DEBUG(this, "timed out writing atr {}", atr_id_->key());
                 throw transaction_operation_failed(ec, e.what()).no_rollback().expired();
             default:
-                debug("retrying atr_rollback_complete");
+                CB_ATTEMPT_CTX_LOG_DEBUG(this, "retrying atr_rollback_complete");
                 throw retry_operation(e.what());
         }
     }
@@ -1613,7 +1771,7 @@ void
 attempt_context_impl::rollback()
 {
     op_list_.wait_and_block_ops();
-    debug("rolling back {}", id());
+    CB_ATTEMPT_CTX_LOG_DEBUG(this, "rolling back {}", id());
     if (op_list_.get_mode().is_query()) {
         auto barrier = std::make_shared<std::promise<void>>();
         auto f = barrier->get_future();
@@ -1630,13 +1788,13 @@ attempt_context_impl::rollback()
     check_expiry_during_commit_or_rollback(STAGE_ROLLBACK, std::nullopt);
     if (!atr_id_ || atr_id_->key().empty() || state() == attempt_state::NOT_STARTED) {
         // TODO: check this, but if we try to rollback an empty txn, we should prevent a subsequent commit
-        debug("rollback called on txn with no mutations");
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "rollback called on txn with no mutations");
         is_done_ = true;
         return;
     }
     if (is_done()) {
         std::string msg("Transaction already done, cannot rollback");
-        error(msg);
+        CB_ATTEMPT_CTX_LOG_ERROR(this, msg);
         // need to raise a FAIL_OTHER which is not retryable or rollback-able
         throw transaction_operation_failed(FAIL_OTHER, msg).no_rollback();
     }
@@ -1644,14 +1802,14 @@ attempt_context_impl::rollback()
         // (1) atr_abort
         retry_op_exp<void>([&]() { atr_abort(); });
         // (2) rollback staged mutations
-        staged_mutations_->rollback(*this);
-        debug("rollback completed unstaging docs");
+        staged_mutations_->rollback(this);
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "rollback completed unstaging docs");
 
         // (3) atr_rollback
         retry_op_exp<void>([&]() { atr_rollback_complete(); });
     } catch (const client_error& e) {
         error_class ec = e.ec();
-        error("rollback transaction {}, attempt {} fail with error {}", transaction_id(), id(), e.what());
+        CB_ATTEMPT_CTX_LOG_ERROR(this, "rollback transaction {}, attempt {} fail with error {}", transaction_id(), id(), e.what());
         if (ec == FAIL_HARD) {
             throw transaction_operation_failed(ec, e.what()).no_rollback();
         }
@@ -1664,10 +1822,10 @@ attempt_context_impl::has_expired_client_side(std::string place, std::optional<c
     bool over = overall_.has_expired_client_side();
     bool hook = hooks_.has_expired_client_side(this, place, doc_id);
     if (over) {
-        debug("{} expired in {}", id(), place);
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "{} expired in {}", id(), place);
     }
     if (hook) {
-        debug("{} fake expiry in {}", id(), place);
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "{} fake expiry in {}", id(), place);
     }
     return over || hook;
 }
@@ -1676,7 +1834,8 @@ bool
 attempt_context_impl::check_expiry_pre_commit(std::string stage, std::optional<const std::string> doc_id)
 {
     if (has_expired_client_side(stage, std::move(doc_id))) {
-        debug("{} has expired in stage {}, entering expiry-overtime mode - will make one attempt to rollback", id(), stage);
+        CB_ATTEMPT_CTX_LOG_DEBUG(
+          this, "{} has expired in stage {}, entering expiry-overtime mode - will make one attempt to rollback", id(), stage);
 
         // [EXP-ROLLBACK] Combo of setting this mode and throwing AttemptExpired will result in an attempt to rollback, which will
         // ignore expiry, and bail out if anything fails
@@ -1690,11 +1849,11 @@ std::optional<error_class>
 attempt_context_impl::error_if_expired_and_not_in_overtime(const std::string& stage, std::optional<const std::string> doc_id)
 {
     if (expiry_overtime_mode_.load()) {
-        debug("not doing expired check in {} as already in expiry-overtime", stage);
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "not doing expired check in {} as already in expiry-overtime", stage);
         return {};
     }
     if (has_expired_client_side(stage, std::move(doc_id))) {
-        debug("expired in {}", stage);
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "expired in {}", stage);
         return FAIL_EXPIRY;
     }
     return {};
@@ -1706,11 +1865,12 @@ attempt_context_impl::check_expiry_during_commit_or_rollback(const std::string&
     // [EXP-COMMIT-OVERTIME]
     if (!expiry_overtime_mode_.load()) {
         if (has_expired_client_side(stage, std::move(doc_id))) {
-            debug("{} has expired in stage {}, entering expiry-overtime mode (one attempt to complete commit)", id(), stage);
+            CB_ATTEMPT_CTX_LOG_DEBUG(
+              this, "{} has expired in stage {}, entering expiry-overtime mode (one attempt to complete commit)", id(), stage);
             expiry_overtime_mode_ = true;
         }
     } else {
-        debug("{} ignoring expiry in stage {}  as in expiry-overtime mode", id(), stage);
+        CB_ATTEMPT_CTX_LOG_DEBUG(this, "{} ignoring expiry in stage {}  as in expiry-overtime mode", id(), stage);
     }
 }
 template<typename Handler>
@@ -1729,7 +1889,7 @@ attempt_context_impl::set_atr_pending_locked(const core::document_id& id, std::u
             auto error_handler =
               [this, &lock](error_class ec, const std::string& message, const core::document_id& doc_id, Handler&& fn) mutable {
                   transaction_operation_failed err(ec, message);
-                  trace("got {} trying to set atr to pending", message);
+                  CB_ATTEMPT_CTX_LOG_TRACE(this, "got {} trying to set atr to pending", message);
                   if (expiry_overtime_mode_.load()) {
                       return fn(err.no_rollback().expired());
                   }
@@ -1747,7 +1907,7 @@ attempt_context_impl::set_atr_pending_locked(const core::document_id& id, std::u
                           // Retry just this
                           overall_.retry_delay();
                           // keep it locked!
-                          debug("got {}, retrying set atr pending", ec);
+                          CB_ATTEMPT_CTX_LOG_DEBUG(this, "got {}, retrying set atr pending", ec);
                           return set_atr_pending_locked(doc_id, std::move(lock), std::move(fn));
                       case FAIL_TRANSIENT:
                           // Retry txn
@@ -1761,14 +1921,14 @@ attempt_context_impl::set_atr_pending_locked(const core::document_id& id, std::u
             if (auto ec = hooks_.before_atr_pending(this); ec) {
                 return error_handler(*ec, "before_atr_pending hook raised error", id, std::forward<Handler>(fn));
             }
-            debug("updating atr {}", atr_id_.value());
+            CB_ATTEMPT_CTX_LOG_DEBUG(this, "updating atr {}", atr_id_.value());
 
             // FIXME: do we need to capture "now" here?
             // std::chrono::time_point<std::chrono::steady_clock> now = std::chrono::steady_clock::now();
             std::chrono::nanoseconds remaining = overall_.remaining();
             // This bounds the value to [0-expirationTime].  It should always be in this range, this is just to protect
             // against the application clock changing.
-            long remaining_bounded_nanos = std::max(std::min(remaining.count(), overall_.config().expiration_time().count()),
+            long remaining_bounded_nanos = std::max(std::min(remaining.count(), overall_.config().expiration_time.count()),
                                                     static_cast<std::chrono::nanoseconds::rep>(0));
             long remaining_bounded_msecs = remaining_bounded_nanos / 1'000'000;
 
@@ -1786,12 +1946,12 @@ attempt_context_impl::set_atr_pending_locked(const core::document_id& id, std::u
                   couchbase::mutate_in_specs::insert(prefix + ATR_FIELD_EXPIRES_AFTER_MSECS, remaining_bounded_msecs).xattr().create_path(),
                   // ExtStoreDurability
                   couchbase::mutate_in_specs::insert(prefix + ATR_FIELD_DURABILITY_LEVEL,
-                                                     store_durability_level_to_string(overall_.config().durability_level()))
+                                                     store_durability_level_to_string(overall_.config().level))
                     .xattr()
                     .create_path(),
                   // subdoc::opcode::set_doc used in replace w/ empty path
                   // ExtBinaryMetadata
-                  couchbase::mutate_in_specs::replace({}, std::string({ 0x00 })),
+                  couchbase::mutate_in_specs::replace_raw({}, std::vector<std::byte>{ std::byte{ 0x00 } }),
               }
                 .specs();
             req.store_semantics = couchbase::store_semantics::upsert;
@@ -1805,7 +1965,7 @@ attempt_context_impl::set_atr_pending_locked(const core::document_id& id, std::u
                   }
                   if (!ec) {
                       state(attempt_state::PENDING);
-                      debug("set ATR {} to Pending, got CAS (start time) {}", atr_id_.value(), resp.cas.value());
+                      CB_ATTEMPT_CTX_LOG_DEBUG(this, "set ATR {} to Pending, got CAS (start time) {}", atr_id_.value(), resp.cas.value());
                       return fn(std::nullopt);
                   }
                   return error_handler(*ec,
@@ -1815,7 +1975,7 @@ attempt_context_impl::set_atr_pending_locked(const core::document_id& id, std::u
               });
         }
     } catch (const std::exception& e) {
-        error("unexpected error setting atr pending {}", e.what());
+        CB_ATTEMPT_CTX_LOG_ERROR(this, "unexpected error setting atr pending {}", e.what());
         return fn(transaction_operation_failed(FAIL_OTHER, "unexpected error setting atr pending"));
     }
 }
@@ -1856,13 +2016,13 @@ attempt_context_impl::do_get(const core::document_id& id, const std::optional<st
 
         staged_mutation* own_write = check_for_own_write(id);
         if (own_write) {
-            debug("found own-write of mutated doc {}", id);
+            CB_ATTEMPT_CTX_LOG_DEBUG(this, "found own-write of mutated doc {}", id);
             return cb(std::nullopt, std::nullopt, transaction_get_result::create_from(own_write->doc(), own_write->content()));
         }
         staged_mutation* own_remove = staged_mutations_->find_remove(id);
         if (own_remove) {
             auto msg = fmt::format("found own-write of removed doc {}", id);
-            debug(msg);
+            CB_ATTEMPT_CTX_LOG_DEBUG(this, msg);
             return cb(FAIL_DOC_NOT_FOUND, msg, std::nullopt);
         }
 
@@ -1880,11 +2040,12 @@ attempt_context_impl::do_get(const core::document_id& id, const std::optional<st
               }
               if (!ec) {
                   if (doc->links().is_document_in_transaction()) {
-                      debug("doc {} in transaction, resolving_missing_atr_entry={}", *doc, resolving_missing_atr_entry.value_or("-"));
+                      CB_ATTEMPT_CTX_LOG_DEBUG(
+                        this, "doc {} in transaction, resolving_missing_atr_entry={}", *doc, resolving_missing_atr_entry.value_or("-"));
 
                       if (resolving_missing_atr_entry.has_value() &&
                           resolving_missing_atr_entry.value() == doc->links().staged_attempt_id()) {
-                          debug("doc is in lost pending transaction");
+                          CB_ATTEMPT_CTX_LOG_DEBUG(this, "doc is in lost pending transaction");
 
                           if (doc->links().is_document_being_inserted()) {
                               // this document is being inserted, so should not be visible yet
@@ -1942,8 +2103,9 @@ attempt_context_impl::do_get(const core::document_id& id, const std::optional<st
                                     }
                                 } else {
                                     // failed to get the ATR entry
-                                    debug("could not get ATR entry, checking again with {}",
-                                          doc->links().staged_attempt_id().value_or("-"));
+                                    CB_ATTEMPT_CTX_LOG_DEBUG(this,
+                                                             "could not get ATR entry, checking again with {}",
+                                                             doc->links().staged_attempt_id().value_or("-"));
                                     return do_get(id, doc->links().staged_attempt_id(), cb);
                                 }
                                 if (ignore_doc) {
@@ -1953,13 +2115,14 @@ attempt_context_impl::do_get(const core::document_id& id, const std::optional<st
 
                             } else {
                                 // failed to get the ATR
-                                debug("could not get ATR, checking again with {}", doc->links().staged_attempt_id().value_or("-"));
+                                CB_ATTEMPT_CTX_LOG_DEBUG(
+                                  this, "could not get ATR, checking again with {}", doc->links().staged_attempt_id().value_or("-"));
                                 return do_get(id, doc->links().staged_attempt_id(), cb);
                             }
                         });
                   } else {
                       if (doc->links().is_deleted()) {
-                          debug("doc not in txn, and is_deleted, so not returning it.");
+                          CB_ATTEMPT_CTX_LOG_DEBUG(this, "doc not in txn, and is_deleted, so not returning it.");
                           // doc has been deleted, not in txn, so don't return it
                           return cb(std::nullopt, std::nullopt, std::nullopt);
                       }
@@ -1982,7 +2145,7 @@ attempt_context_impl::do_get(const core::document_id& id, const std::optional<st
 void
 attempt_context_impl::get_doc(
   const core::document_id& id,
-  utils::movable_function<void(std::optional<error_class>, std::optional<std::string>, std::optional<transaction_get_result>)>&& cb)
+  std::function<void(std::optional<error_class>, std::optional<std::string>, std::optional<transaction_get_result>)>&& cb)
 {
     core::operations::lookup_in_request req{ id };
     req.specs =
@@ -2008,7 +2171,7 @@ attempt_context_impl::get_doc(
         overall_.cluster_ref()->execute(req, [this, id, cb = std::move(cb)](core::operations::lookup_in_response resp) {
             auto ec = error_class_from_response(resp);
             if (ec) {
-                trace("get_doc got error {} : {}", resp.ctx.ec().message(), *ec);
+                CB_ATTEMPT_CTX_LOG_TRACE(this, "get_doc got error {} : {}", resp.ctx.ec().message(), *ec);
                 switch (*ec) {
                     case FAIL_PATH_NOT_FOUND:
                         return cb(*ec, resp.ctx.ec().message(), transaction_get_result::create_from(resp));
@@ -2034,7 +2197,7 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
                                                          error_class ec,
                                                          const std::string& message)
 {
-    trace("create_staged_insert got error class {}: {}", ec, message);
+    CB_ATTEMPT_CTX_LOG_TRACE(this, "create_staged_insert got error class {}: {}", ec, message);
     if (expiry_overtime_mode_.load()) {
         return op_completed_with_error(std::forward<Handler>(cb), transaction_operation_failed(FAIL_EXPIRY, "attempt timed out").expired());
     }
@@ -2046,7 +2209,7 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
             return op_completed_with_error(std::forward<Handler>(cb),
                                            transaction_operation_failed(ec, "transient error in insert").retry());
         case FAIL_AMBIGUOUS:
-            debug("FAIL_AMBIGUOUS in create_staged_insert, retrying");
+            CB_ATTEMPT_CTX_LOG_DEBUG(this, "FAIL_AMBIGUOUS in create_staged_insert, retrying");
             delay();
             return create_staged_insert(id, content, cas, delay, std::forward<Handler>(cb));
         case FAIL_OTHER:
@@ -2057,9 +2220,10 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
         case FAIL_DOC_ALREADY_EXISTS:
         case FAIL_CAS_MISMATCH: {
             // special handling for doc already existing
-            debug("found existing doc {}, may still be able to insert", id);
+            CB_ATTEMPT_CTX_LOG_DEBUG(this, "found existing doc {}, may still be able to insert", id);
             auto error_handler = [this, id, content](error_class ec2, const std::string& err_message, Handler&& cb) mutable {
-                trace("after a CAS_MISMATCH or DOC_ALREADY_EXISTS, then got error {} in create_staged_insert", ec2);
+                CB_ATTEMPT_CTX_LOG_TRACE(
+                  this, "after a CAS_MISMATCH or DOC_ALREADY_EXISTS, then got error {} in create_staged_insert", ec2);
                 if (expiry_overtime_mode_.load()) {
                     return op_completed_with_error(std::move(cb), transaction_operation_failed(FAIL_EXPIRY, "attempt timed out").expired());
                 }
@@ -2087,10 +2251,11 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
                 std::optional<error_class> ec3, std::optional<std::string> err_message, std::optional<transaction_get_result> doc) mutable {
                   if (!ec3) {
                       if (doc) {
-                          debug("document {} exists, is_in_transaction {}, is_deleted {} ",
-                                doc->id(),
-                                doc->links().is_document_in_transaction(),
-                                doc->links().is_deleted());
+                          CB_ATTEMPT_CTX_LOG_DEBUG(this,
+                                                   "document {} exists, is_in_transaction {}, is_deleted {} ",
+                                                   doc->id(),
+                                                   doc->links().is_document_in_transaction(),
+                                                   doc->links().is_deleted());
 
                           if (auto err = forward_compat::check(forward_compat_stage::WWC_INSERTING_GET, doc->links().forward_compat());
                               err) {
@@ -2098,16 +2263,17 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
                           }
                           if (!doc->links().is_document_in_transaction() && doc->links().is_deleted()) {
                               // it is just a deleted doc, so we are ok.  Let's try again, but with the cas
-                              debug("create staged insert found existing deleted doc, retrying with cas {}", doc->cas().value());
+                              CB_ATTEMPT_CTX_LOG_DEBUG(
+                                this, "create staged insert found existing deleted doc, retrying with cas {}", doc->cas().value());
                               delay();
                               return create_staged_insert(id, content, doc->cas().value(), delay, std::forward<Handler>(cb));
                           }
                           if (!doc->links().is_document_in_transaction()) {
                               // doc was inserted outside txn elsewhere
-                              trace("doc {} not in txn - was inserted outside txn", id);
+                              CB_ATTEMPT_CTX_LOG_TRACE(this, "doc {} not in txn - was inserted outside txn", id);
                               return op_completed_with_error(
                                 std::forward<Handler>(cb),
-                                transaction_operation_failed(FAIL_DOC_ALREADY_EXISTS, "document already exists"));
+                                document_exists({ couchbase::errc::transaction_op::document_exists_exception, key_value_error_context() }));
                           }
                           // CBD-3787 - Only a staged insert is ok to overwrite
                           if (doc->links().op() && *doc->links().op() != "insert") {
@@ -2124,13 +2290,14 @@ attempt_context_impl::create_staged_insert_error_handler(const core::document_id
                                 if (err) {
                                     return op_completed_with_error(std::move(cb), *err);
                                 }
-                                debug("doc ok to overwrite, retrying create_staged_insert with cas {}", doc->cas().value());
+                                CB_ATTEMPT_CTX_LOG_DEBUG(
+                                  this, "doc ok to overwrite, retrying create_staged_insert with cas {}", doc->cas().value());
                                 delay();
                                 return create_staged_insert(id, content, doc->cas().value(), delay, std::forward<Handler>(cb));
                             });
                       } else {
                           // no doc now, just retry entire txn
-                          trace("got {} from get_doc in exists during staged insert", *ec3);
+                          CB_ATTEMPT_CTX_LOG_TRACE(this, "got {} from get_doc in exists during staged insert", *ec3);
                           return op_completed_with_error(
                             std::move(cb),
                             transaction_operation_failed(FAIL_DOC_NOT_FOUND, "insert failed as the doc existed, but now seems to not exist")
@@ -2165,7 +2332,7 @@ attempt_context_impl::create_staged_insert(const core::document_id& id,
         return create_staged_insert_error_handler(
           id, content, cas, std::forward<Delay>(delay), std::forward<Handler>(cb), *ec, "before_staged_insert hook threw error");
     }
-    debug("about to insert staged doc {} with cas {}", id, cas);
+    CB_ATTEMPT_CTX_LOG_DEBUG(this, "about to insert staged doc {} with cas {}", id, cas);
     auto req = create_staging_request(id, nullptr, "insert", content);
     req.access_deleted = true;
     req.create_as_deleted = true;
@@ -2181,7 +2348,7 @@ attempt_context_impl::create_staged_insert(const core::document_id& id,
                 id, content, cas, std::forward<Delay>(delay), std::forward<Handler>(cb), *ec, "after_staged_insert hook threw error");
           }
           if (!resp.ctx.ec()) {
-              debug("inserted doc {} CAS={}, {}", id, resp.cas.value(), resp.ctx.ec().message());
+              CB_ATTEMPT_CTX_LOG_DEBUG(this, "inserted doc {} CAS={}, {}", id, resp.cas.value(), resp.ctx.ec().message());
 
               // TODO: clean this up (do most of this in transactions_document(...))
               transaction_links links(atr_id_->key(),