couchbase 3.0.0.beta.1 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (103) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +227 -0
  3. data/.rubocop_todo.yml +47 -0
  4. data/CONTRIBUTING.md +110 -0
  5. data/Gemfile +4 -0
  6. data/README.md +3 -3
  7. data/Rakefile +1 -1
  8. data/couchbase.gemspec +40 -39
  9. data/examples/analytics.rb +123 -108
  10. data/examples/auth.rb +33 -0
  11. data/examples/crud.rb +16 -2
  12. data/examples/managing_analytics_indexes.rb +18 -4
  13. data/examples/managing_buckets.rb +17 -3
  14. data/examples/managing_collections.rb +22 -9
  15. data/examples/managing_query_indexes.rb +38 -18
  16. data/examples/managing_search_indexes.rb +21 -6
  17. data/examples/managing_view_indexes.rb +18 -4
  18. data/examples/query.rb +17 -3
  19. data/examples/query_with_consistency.rb +30 -20
  20. data/examples/search.rb +116 -101
  21. data/examples/search_with_consistency.rb +43 -30
  22. data/examples/subdocument.rb +42 -30
  23. data/examples/view.rb +19 -10
  24. data/ext/CMakeLists.txt +40 -2
  25. data/ext/build_version.hxx.in +1 -1
  26. data/ext/couchbase/bucket.hxx +190 -38
  27. data/ext/couchbase/cluster.hxx +22 -4
  28. data/ext/couchbase/configuration.hxx +14 -14
  29. data/ext/couchbase/couchbase.cxx +108 -12
  30. data/ext/couchbase/error_map.hxx +202 -2
  31. data/ext/couchbase/errors.hxx +8 -2
  32. data/ext/couchbase/io/dns_client.hxx +6 -6
  33. data/ext/couchbase/io/http_command.hxx +2 -2
  34. data/ext/couchbase/io/http_session.hxx +7 -11
  35. data/ext/couchbase/io/http_session_manager.hxx +3 -3
  36. data/ext/couchbase/io/mcbp_command.hxx +101 -44
  37. data/ext/couchbase/io/mcbp_session.hxx +144 -49
  38. data/ext/couchbase/io/retry_action.hxx +30 -0
  39. data/ext/couchbase/io/retry_context.hxx +39 -0
  40. data/ext/couchbase/io/retry_orchestrator.hxx +96 -0
  41. data/ext/couchbase/io/retry_reason.hxx +235 -0
  42. data/ext/couchbase/io/retry_strategy.hxx +156 -0
  43. data/ext/couchbase/operations/document_decrement.hxx +2 -0
  44. data/ext/couchbase/operations/document_exists.hxx +2 -0
  45. data/ext/couchbase/operations/document_get.hxx +2 -0
  46. data/ext/couchbase/operations/document_get_and_lock.hxx +2 -0
  47. data/ext/couchbase/operations/document_get_and_touch.hxx +2 -0
  48. data/ext/couchbase/operations/document_get_projected.hxx +2 -0
  49. data/ext/couchbase/operations/document_increment.hxx +2 -0
  50. data/ext/couchbase/operations/document_insert.hxx +2 -0
  51. data/ext/couchbase/operations/document_lookup_in.hxx +2 -0
  52. data/ext/couchbase/operations/document_mutate_in.hxx +3 -0
  53. data/ext/couchbase/operations/document_query.hxx +10 -0
  54. data/ext/couchbase/operations/document_remove.hxx +2 -0
  55. data/ext/couchbase/operations/document_replace.hxx +2 -0
  56. data/ext/couchbase/operations/document_search.hxx +8 -3
  57. data/ext/couchbase/operations/document_touch.hxx +2 -0
  58. data/ext/couchbase/operations/document_unlock.hxx +2 -0
  59. data/ext/couchbase/operations/document_upsert.hxx +2 -0
  60. data/ext/couchbase/operations/query_index_create.hxx +14 -4
  61. data/ext/couchbase/operations/query_index_drop.hxx +12 -2
  62. data/ext/couchbase/operations/query_index_get_all.hxx +11 -2
  63. data/ext/couchbase/origin.hxx +47 -17
  64. data/ext/couchbase/platform/backtrace.c +189 -0
  65. data/ext/couchbase/platform/backtrace.h +54 -0
  66. data/ext/couchbase/platform/terminate_handler.cc +122 -0
  67. data/ext/couchbase/platform/terminate_handler.h +36 -0
  68. data/ext/couchbase/protocol/cmd_get_cluster_config.hxx +6 -1
  69. data/ext/couchbase/protocol/status.hxx +14 -4
  70. data/ext/couchbase/version.hxx +2 -2
  71. data/ext/extconf.rb +39 -36
  72. data/ext/test/main.cxx +64 -16
  73. data/lib/couchbase.rb +0 -1
  74. data/lib/couchbase/analytics_options.rb +2 -4
  75. data/lib/couchbase/authenticator.rb +14 -0
  76. data/lib/couchbase/binary_collection.rb +9 -9
  77. data/lib/couchbase/binary_collection_options.rb +8 -6
  78. data/lib/couchbase/bucket.rb +18 -18
  79. data/lib/couchbase/cluster.rb +121 -90
  80. data/lib/couchbase/collection.rb +36 -38
  81. data/lib/couchbase/collection_options.rb +31 -17
  82. data/lib/couchbase/common_options.rb +1 -1
  83. data/lib/couchbase/datastructures/couchbase_list.rb +16 -16
  84. data/lib/couchbase/datastructures/couchbase_map.rb +18 -18
  85. data/lib/couchbase/datastructures/couchbase_queue.rb +13 -13
  86. data/lib/couchbase/datastructures/couchbase_set.rb +8 -7
  87. data/lib/couchbase/errors.rb +10 -3
  88. data/lib/couchbase/json_transcoder.rb +2 -2
  89. data/lib/couchbase/management/analytics_index_manager.rb +37 -37
  90. data/lib/couchbase/management/bucket_manager.rb +25 -25
  91. data/lib/couchbase/management/collection_manager.rb +3 -3
  92. data/lib/couchbase/management/query_index_manager.rb +59 -14
  93. data/lib/couchbase/management/search_index_manager.rb +15 -12
  94. data/lib/couchbase/management/user_manager.rb +1 -1
  95. data/lib/couchbase/management/view_index_manager.rb +11 -5
  96. data/lib/couchbase/mutation_state.rb +12 -0
  97. data/lib/couchbase/query_options.rb +23 -9
  98. data/lib/couchbase/scope.rb +61 -1
  99. data/lib/couchbase/search_options.rb +40 -27
  100. data/lib/couchbase/subdoc.rb +31 -28
  101. data/lib/couchbase/version.rb +2 -2
  102. data/lib/couchbase/view_options.rb +0 -1
  103. metadata +22 -9
@@ -247,7 +247,9 @@ enum class analytics_errc {
247
247
  };
248
248
 
249
249
  /// Errors related to Search service (CBFT)
250
- enum class search_errc {};
250
+ enum class search_errc {
251
+ index_not_ready = 400,
252
+ };
251
253
 
252
254
  /// Errors related to Views service (CAPI)
253
255
  enum class view_errc {
@@ -465,8 +467,12 @@ struct search_error_category : std::error_category {
465
467
  return "search";
466
468
  }
467
469
 
468
- [[nodiscard]] std::string message(int) const noexcept override
470
+ [[nodiscard]] std::string message(int ev) const noexcept override
469
471
  {
472
+ switch (static_cast<search_errc>(ev)) {
473
+ case search_errc::index_not_ready:
474
+ return "index_not_ready";
475
+ }
470
476
  return "FIXME: unknown error code in search category (recompile with newer library)";
471
477
  }
472
478
  };
@@ -81,18 +81,17 @@ class dns_client
81
81
  std::size_t /* bytes_transferred */) mutable {
82
82
  if (ec1 == asio::error::operation_aborted) {
83
83
  self->deadline_.cancel();
84
- return handler({ std::make_error_code(error::common_errc::ambiguous_timeout) });
84
+ return handler({ std::make_error_code(error::common_errc::unambiguous_timeout) });
85
85
  }
86
86
  if (ec1) {
87
87
  self->deadline_.cancel();
88
88
  return handler({ ec1 });
89
89
  }
90
90
 
91
- asio::ip::udp::endpoint sender_endpoint;
92
91
  self->recv_buf_.resize(512);
93
92
  self->udp_.async_receive_from(
94
93
  asio::buffer(self->recv_buf_),
95
- sender_endpoint,
94
+ self->udp_sender_,
96
95
  [self, handler = std::forward<Handler>(handler)](std::error_code ec2, std::size_t bytes_transferred) mutable {
97
96
  self->deadline_.cancel();
98
97
  if (ec2) {
@@ -149,7 +148,7 @@ class dns_client
149
148
  if (ec2) {
150
149
  self->deadline_.cancel();
151
150
  if (ec2 == asio::error::operation_aborted) {
152
- ec2 = std::make_error_code(error::common_errc::ambiguous_timeout);
151
+ ec2 = std::make_error_code(error::common_errc::unambiguous_timeout);
153
152
  }
154
153
  return handler({ ec2 });
155
154
  }
@@ -189,14 +188,15 @@ class dns_client
189
188
 
190
189
  asio::steady_timer deadline_;
191
190
  asio::ip::udp::socket udp_;
191
+ asio::ip::udp::endpoint udp_sender_{};
192
192
  asio::ip::tcp::socket tcp_;
193
193
 
194
194
  asio::ip::address address_;
195
195
  std::uint16_t port_;
196
196
 
197
- std::vector<uint8_t> send_buf_;
197
+ std::vector<uint8_t> send_buf_{};
198
198
  std::uint16_t recv_buf_size_{ 0 };
199
- std::vector<uint8_t> recv_buf_;
199
+ std::vector<uint8_t> recv_buf_{};
200
200
  };
201
201
 
202
202
  explicit dns_client(asio::io_context& ctx)
@@ -45,7 +45,7 @@ struct http_command : public std::enable_shared_from_this<http_command<Request>>
45
45
  request.encode_to(encoded);
46
46
  encoded.headers["client-context-id"] = request.client_context_id;
47
47
  auto log_prefix = session->log_prefix();
48
- spdlog::debug("{} HTTP request: {}, method={}, path={}, client_context_id={}, timeout={}ms",
48
+ spdlog::trace("{} HTTP request: {}, method={}, path={}, client_context_id={}, timeout={}ms",
49
49
  log_prefix,
50
50
  encoded.type,
51
51
  encoded.method,
@@ -65,7 +65,7 @@ struct http_command : public std::enable_shared_from_this<http_command<Request>>
65
65
  std::error_code ec, io::http_response&& msg) mutable {
66
66
  self->deadline.cancel();
67
67
  encoded_response_type resp(msg);
68
- spdlog::debug("{} HTTP response: {}, client_context_id={}, status={}",
68
+ spdlog::trace("{} HTTP response: {}, client_context_id={}, status={}",
69
69
  log_prefix,
70
70
  self->request.type,
71
71
  self->request.client_context_id,
@@ -43,8 +43,7 @@ class http_session : public std::enable_shared_from_this<http_session>
43
43
  public:
44
44
  http_session(const std::string& client_id,
45
45
  asio::io_context& ctx,
46
- const std::string& username,
47
- const std::string& password,
46
+ const cluster_credentials& credentials,
48
47
  const std::string& hostname,
49
48
  const std::string& service)
50
49
  : client_id_(client_id)
@@ -53,8 +52,7 @@ class http_session : public std::enable_shared_from_this<http_session>
53
52
  , resolver_(ctx_)
54
53
  , stream_(std::make_unique<plain_stream_impl>(ctx_))
55
54
  , deadline_timer_(ctx_)
56
- , username_(username)
57
- , password_(password)
55
+ , credentials_(credentials)
58
56
  , hostname_(hostname)
59
57
  , service_(service)
60
58
  , user_agent_(fmt::format("ruby/{}.{}.{}/{}; client/{}; session/{}; {}",
@@ -72,8 +70,7 @@ class http_session : public std::enable_shared_from_this<http_session>
72
70
  http_session(const std::string& client_id,
73
71
  asio::io_context& ctx,
74
72
  asio::ssl::context& tls,
75
- const std::string& username,
76
- const std::string& password,
73
+ const cluster_credentials& credentials,
77
74
  const std::string& hostname,
78
75
  const std::string& service)
79
76
  : client_id_(client_id)
@@ -82,8 +79,7 @@ class http_session : public std::enable_shared_from_this<http_session>
82
79
  , resolver_(ctx_)
83
80
  , stream_(std::make_unique<tls_stream_impl>(ctx_, tls))
84
81
  , deadline_timer_(ctx_)
85
- , username_(username)
86
- , password_(password)
82
+ , credentials_(credentials)
87
83
  , hostname_(hostname)
88
84
  , service_(service)
89
85
  , user_agent_(fmt::format("ruby/{}.{}.{}/{}; client/{}; session/{}; {}",
@@ -191,7 +187,8 @@ class http_session : public std::enable_shared_from_this<http_session>
191
187
  return;
192
188
  }
193
189
  request.headers["user-agent"] = user_agent_;
194
- request.headers["authorization"] = fmt::format("Basic {}", base64::encode(fmt::format("{}:{}", username_, password_)));
190
+ request.headers["authorization"] =
191
+ fmt::format("Basic {}", base64::encode(fmt::format("{}:{}", credentials_.username, credentials_.password)));
195
192
  write(fmt::format("{} {} HTTP/1.1\r\nhost: {}:{}\r\n", request.method, request.path, hostname_, service_));
196
193
  if (!request.body.empty()) {
197
194
  request.headers["content-length"] = std::to_string(request.body.size());
@@ -335,8 +332,7 @@ class http_session : public std::enable_shared_from_this<http_session>
335
332
  std::unique_ptr<stream_impl> stream_;
336
333
  asio::steady_timer deadline_timer_;
337
334
 
338
- std::string username_;
339
- std::string password_;
335
+ cluster_credentials credentials_;
340
336
  std::string hostname_;
341
337
  std::string service_;
342
338
  std::string user_agent_;
@@ -48,7 +48,7 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
48
48
  }
49
49
  }
50
50
 
51
- std::shared_ptr<http_session> check_out(service_type type, const std::string& username, const std::string& password)
51
+ std::shared_ptr<http_session> check_out(service_type type, const couchbase::cluster_credentials& credentials)
52
52
  {
53
53
  std::scoped_lock lock(sessions_mutex_);
54
54
  if (idle_sessions_[type].empty()) {
@@ -61,9 +61,9 @@ class http_session_manager : public std::enable_shared_from_this<http_session_ma
61
61
  config_.nodes.size();
62
62
  std::shared_ptr<http_session> session;
63
63
  if (options_.enable_tls) {
64
- session = std::make_shared<http_session>(client_id_, ctx_, tls_, username, password, hostname, std::to_string(port));
64
+ session = std::make_shared<http_session>(client_id_, ctx_, tls_, credentials, hostname, std::to_string(port));
65
65
  } else {
66
- session = std::make_shared<http_session>(client_id_, ctx_, username, password, hostname, std::to_string(port));
66
+ session = std::make_shared<http_session>(client_id_, ctx_, credentials, hostname, std::to_string(port));
67
67
  }
68
68
  session->start();
69
69
  session->on_stop([type, id = session->id(), self = this->shared_from_this()]() {
@@ -18,6 +18,8 @@
18
18
  #pragma once
19
19
 
20
20
  #include <io/mcbp_session.hxx>
21
+ #include <io/retry_orchestrator.hxx>
22
+
21
23
  #include <protocol/cmd_get_collection_id.hxx>
22
24
  #include <functional>
23
25
  #include <utility>
@@ -27,8 +29,8 @@ namespace couchbase::operations
27
29
 
28
30
  using mcbp_command_handler = std::function<void(std::error_code, std::optional<io::mcbp_message>)>;
29
31
 
30
- template<typename Request>
31
- struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>> {
32
+ template<typename Manager, typename Request>
33
+ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Manager, Request>> {
32
34
  using encoded_request_type = typename Request::encoded_request_type;
33
35
  using encoded_response_type = typename Request::encoded_response_type;
34
36
  asio::steady_timer deadline;
@@ -38,11 +40,13 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>>
38
40
  std::optional<std::uint32_t> opaque_{};
39
41
  std::shared_ptr<io::mcbp_session> session_{};
40
42
  mcbp_command_handler handler_{};
43
+ std::shared_ptr<Manager> manager_{};
41
44
 
42
- mcbp_command(asio::io_context& ctx, Request req)
45
+ mcbp_command(asio::io_context& ctx, std::shared_ptr<Manager> manager, Request req)
43
46
  : deadline(ctx)
44
47
  , retry_backoff(ctx)
45
48
  , request(req)
49
+ , manager_(manager)
46
50
  {
47
51
  }
48
52
 
@@ -54,16 +58,21 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>>
54
58
  if (ec == asio::error::operation_aborted) {
55
59
  return;
56
60
  }
57
- self->cancel();
61
+ self->cancel(io::retry_reason::do_not_retry);
58
62
  });
59
63
  }
60
64
 
61
- void cancel()
65
+ void cancel(io::retry_reason reason)
62
66
  {
63
67
  if (opaque_ && session_) {
64
- session_->cancel(opaque_.value(), asio::error::operation_aborted);
68
+ if (session_->cancel(opaque_.value(), asio::error::operation_aborted, reason)) {
69
+ handler_ = nullptr;
70
+ }
65
71
  }
66
- handler_ = nullptr;
72
+ invoke_handler(std::make_error_code(request.retries.idempotent ? error::common_errc::unambiguous_timeout
73
+ : error::common_errc::ambiguous_timeout));
74
+ retry_backoff.cancel();
75
+ deadline.cancel();
67
76
  }
68
77
 
69
78
  void invoke_handler(std::error_code ec, std::optional<io::mcbp_message> msg = {})
@@ -79,26 +88,27 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>>
79
88
  protocol::client_request<protocol::get_collection_id_request_body> req;
80
89
  req.opaque(session_->next_opaque());
81
90
  req.body().collection_path(request.id.collection);
82
- session_->write_and_subscribe(req.opaque(),
83
- req.data(session_->supports_feature(protocol::hello_feature::snappy)),
84
- [self = this->shared_from_this()](std::error_code ec, io::mcbp_message&& msg) mutable {
85
- if (ec == asio::error::operation_aborted) {
86
- return self->invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
87
- }
88
- if (ec == std::make_error_code(error::common_errc::collection_not_found)) {
89
- if (self->request.id.collection_uid) {
90
- return self->handle_unknown_collection();
91
- }
92
- return self->invoke_handler(ec);
93
- }
94
- if (ec) {
95
- return self->invoke_handler(ec);
96
- }
97
- protocol::client_response<protocol::get_collection_id_response_body> resp(msg);
98
- self->session_->update_collection_uid(self->request.id.collection, resp.body().collection_uid());
99
- self->request.id.collection_uid = resp.body().collection_uid();
100
- return self->send();
101
- });
91
+ session_->write_and_subscribe(
92
+ req.opaque(),
93
+ req.data(session_->supports_feature(protocol::hello_feature::snappy)),
94
+ [self = this->shared_from_this()](std::error_code ec, io::retry_reason /* reason */, io::mcbp_message&& msg) mutable {
95
+ if (ec == asio::error::operation_aborted) {
96
+ return self->invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
97
+ }
98
+ if (ec == std::make_error_code(error::common_errc::collection_not_found)) {
99
+ if (self->request.id.collection_uid) {
100
+ return self->handle_unknown_collection();
101
+ }
102
+ return self->invoke_handler(ec);
103
+ }
104
+ if (ec) {
105
+ return self->invoke_handler(ec);
106
+ }
107
+ protocol::client_response<protocol::get_collection_id_response_body> resp(msg);
108
+ self->session_->update_collection_uid(self->request.id.collection, resp.body().collection_uid());
109
+ self->request.id.collection_uid = resp.body().collection_uid();
110
+ return self->send();
111
+ });
102
112
  }
103
113
 
104
114
  void handle_unknown_collection()
@@ -112,7 +122,8 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>>
112
122
  request.id.key,
113
123
  std::chrono::duration_cast<std::chrono::milliseconds>(time_left).count());
114
124
  if (time_left < backoff) {
115
- return invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
125
+ return invoke_handler(std::make_error_code(request.retries.idempotent ? error::common_errc::unambiguous_timeout
126
+ : error::common_errc::ambiguous_timeout));
116
127
  }
117
128
  retry_backoff.expires_after(backoff);
118
129
  retry_backoff.async_wait([self = this->shared_from_this()](std::error_code ec) mutable {
@@ -149,22 +160,68 @@ struct mcbp_command : public std::enable_shared_from_this<mcbp_command<Request>>
149
160
  }
150
161
  request.encode_to(encoded);
151
162
 
152
- session_->write_and_subscribe(request.opaque,
153
- encoded.data(session_->supports_feature(protocol::hello_feature::snappy)),
154
- [self = this->shared_from_this()](std::error_code ec, io::mcbp_message&& msg) mutable {
155
- self->retry_backoff.cancel();
156
- if (ec == asio::error::operation_aborted) {
157
- return self->invoke_handler(std::make_error_code(error::common_errc::ambiguous_timeout));
158
- }
159
- if (ec == std::make_error_code(error::common_errc::request_canceled)) {
160
- return self->invoke_handler(ec);
161
- }
162
- if (msg.header.status() == static_cast<std::uint16_t>(protocol::status::unknown_collection)) {
163
- return self->handle_unknown_collection();
164
- }
165
- self->deadline.cancel();
166
- self->invoke_handler(ec, msg);
167
- });
163
+ session_->write_and_subscribe(
164
+ request.opaque,
165
+ encoded.data(session_->supports_feature(protocol::hello_feature::snappy)),
166
+ [self = this->shared_from_this()](std::error_code ec, io::retry_reason reason, io::mcbp_message&& msg) mutable {
167
+ self->retry_backoff.cancel();
168
+ if (ec == asio::error::operation_aborted) {
169
+ return self->invoke_handler(std::make_error_code(
170
+ self->request.retries.idempotent ? error::common_errc::unambiguous_timeout : error::common_errc::ambiguous_timeout));
171
+ }
172
+ if (ec == std::make_error_code(error::common_errc::request_canceled)) {
173
+ if (reason == io::retry_reason::do_not_retry) {
174
+ return self->invoke_handler(ec);
175
+ }
176
+ return io::retry_orchestrator::maybe_retry(self->manager_, self, reason, ec);
177
+ }
178
+ protocol::status status = protocol::status::invalid;
179
+ std::optional<error_map::error_info> error_code{};
180
+ if (protocol::is_valid_status(msg.header.status())) {
181
+ status = static_cast<protocol::status>(msg.header.status());
182
+ } else {
183
+ error_code = self->session_->decode_error_code(msg.header.status());
184
+ }
185
+ if (status == protocol::status::not_my_vbucket) {
186
+ self->session_->handle_not_my_vbucket(std::move(msg));
187
+ return io::retry_orchestrator::maybe_retry(self->manager_, self, io::retry_reason::kv_not_my_vbucket, ec);
188
+ }
189
+ if (status == protocol::status::unknown_collection) {
190
+ return self->handle_unknown_collection();
191
+ }
192
+ if (error_code && error_code.value().has_retry_attribute()) {
193
+ reason = io::retry_reason::kv_error_map_retry_indicated;
194
+ } else {
195
+ switch (status) {
196
+ case protocol::status::locked:
197
+ if (encoded_request_type::body_type::opcode != protocol::client_opcode::unlock) {
198
+ /**
199
+ * special case for unlock command, when it should not be retried, because it does not make sense
200
+ * (someone else unlocked the document)
201
+ */
202
+ reason = io::retry_reason::kv_locked;
203
+ }
204
+ break;
205
+ case protocol::status::temporary_failure:
206
+ reason = io::retry_reason::kv_temporary_failure;
207
+ break;
208
+ case protocol::status::sync_write_in_progress:
209
+ reason = io::retry_reason::kv_sync_write_in_progress;
210
+ break;
211
+ case protocol::status::sync_write_re_commit_in_progress:
212
+ reason = io::retry_reason::kv_sync_write_re_commit_in_progress;
213
+ break;
214
+ default:
215
+ break;
216
+ }
217
+ }
218
+ if (reason == io::retry_reason::do_not_retry) {
219
+ self->deadline.cancel();
220
+ self->invoke_handler(ec, msg);
221
+ } else {
222
+ io::retry_orchestrator::maybe_retry(self->manager_, self, reason, ec);
223
+ }
224
+ });
168
225
  }
169
226
 
170
227
  void send_to(std::shared_ptr<io::mcbp_session> session)
@@ -28,6 +28,7 @@
28
28
  #include <io/mcbp_message.hxx>
29
29
  #include <io/mcbp_parser.hxx>
30
30
  #include <io/streams.hxx>
31
+ #include <io/retry_orchestrator.hxx>
31
32
 
32
33
  #include <timeout_defaults.hxx>
33
34
 
@@ -120,8 +121,8 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
120
121
 
121
122
  explicit bootstrap_handler(std::shared_ptr<mcbp_session> session)
122
123
  : session_(session)
123
- , sasl_([origin = session_->origin_]() -> std::string { return origin.get_username(); },
124
- [origin = session_->origin_]() -> std::string { return origin.get_password(); },
124
+ , sasl_([origin = session_->origin_]() -> std::string { return origin.username(); },
125
+ [origin = session_->origin_]() -> std::string { return origin.password(); },
125
126
  { "SCRAM-SHA512", "SCRAM-SHA256", "SCRAM-SHA1", "PLAIN" })
126
127
  {
127
128
  tao::json::value user_agent{
@@ -139,18 +140,20 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
139
140
  fmt::join(hello_req.body().features(), ", "));
140
141
  session_->write(hello_req.data());
141
142
 
142
- protocol::client_request<protocol::sasl_list_mechs_request_body> list_req;
143
- list_req.opaque(session_->next_opaque());
144
- session_->write(list_req.data());
145
-
146
- protocol::client_request<protocol::sasl_auth_request_body> auth_req;
147
- sasl::error sasl_code;
148
- std::string_view sasl_payload;
149
- std::tie(sasl_code, sasl_payload) = sasl_.start();
150
- auth_req.opaque(session_->next_opaque());
151
- auth_req.body().mechanism(sasl_.get_name());
152
- auth_req.body().sasl_data(sasl_payload);
153
- session_->write(auth_req.data());
143
+ if (!session->origin_.credentials().uses_certificate()) {
144
+ protocol::client_request<protocol::sasl_list_mechs_request_body> list_req;
145
+ list_req.opaque(session_->next_opaque());
146
+ session_->write(list_req.data());
147
+
148
+ protocol::client_request<protocol::sasl_auth_request_body> auth_req;
149
+ sasl::error sasl_code;
150
+ std::string_view sasl_payload;
151
+ std::tie(sasl_code, sasl_payload) = sasl_.start();
152
+ auth_req.opaque(session_->next_opaque());
153
+ auth_req.body().mechanism(sasl_.get_name());
154
+ auth_req.body().sasl_data(sasl_payload);
155
+ session_->write(auth_req.data());
156
+ }
154
157
 
155
158
  session_->flush();
156
159
  }
@@ -194,6 +197,10 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
194
197
  if (resp.status() == protocol::status::success) {
195
198
  session_->supported_features_ = resp.body().supported_features();
196
199
  spdlog::debug("{} supported_features=[{}]", session_->log_prefix_, fmt::join(session_->supported_features_, ", "));
200
+ if (session_->origin_.credentials().uses_certificate()) {
201
+ spdlog::debug("{} skip SASL authentication, because TLS certificate was specified", session_->log_prefix_);
202
+ return auth_success();
203
+ }
197
204
  } else {
198
205
  spdlog::warn("{} unexpected message status during bootstrap: {}", session_->log_prefix_, resp.error_message());
199
206
  return complete(std::make_error_code(error::network_errc::handshake_failure));
@@ -246,7 +253,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
246
253
  case protocol::client_opcode::get_error_map: {
247
254
  protocol::client_response<protocol::get_error_map_response_body> resp(msg);
248
255
  if (resp.status() == protocol::status::success) {
249
- session_->errmap_.emplace(resp.body().errmap());
256
+ session_->error_map_.emplace(resp.body().errmap());
250
257
  } else {
251
258
  spdlog::warn("{} unexpected message status during bootstrap: {} (opcode={})",
252
259
  session_->log_prefix_,
@@ -369,16 +376,16 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
369
376
  auto handler = session_->command_handlers_.find(opaque);
370
377
  if (handler != session_->command_handlers_.end()) {
371
378
  auto ec = session_->map_status_code(opcode, status);
372
- spdlog::debug("{} MCBP invoke operation handler, opaque={}, status={}, ec={}",
379
+ spdlog::trace("{} MCBP invoke operation handler: opaque={}, status={}, ec={}",
373
380
  session_->log_prefix_,
374
381
  opaque,
375
- status,
382
+ protocol::status_to_string(status),
376
383
  ec.message());
377
384
  auto fun = handler->second;
378
385
  session_->command_handlers_.erase(handler);
379
- fun(ec, std::move(msg));
386
+ fun(ec, retry_reason::do_not_retry, std::move(msg));
380
387
  } else {
381
- spdlog::debug("{} unexpected orphan response opcode={}, opaque={}",
388
+ spdlog::debug("{} unexpected orphan response: opcode={}, opaque={}",
382
389
  session_->log_prefix_,
383
390
  msg.header.opcode,
384
391
  msg.header.opaque);
@@ -472,7 +479,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
472
479
 
473
480
  ~mcbp_session()
474
481
  {
475
- stop();
482
+ stop(retry_reason::do_not_retry);
476
483
  }
477
484
 
478
485
  [[nodiscard]] const std::string& log_prefix() const
@@ -480,8 +487,9 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
480
487
  return log_prefix_;
481
488
  }
482
489
 
483
- void bootstrap(std::function<void(std::error_code, configuration)>&& handler)
490
+ void bootstrap(std::function<void(std::error_code, configuration)>&& handler, bool retry_on_bucket_not_found = false)
484
491
  {
492
+ retry_bootstrap_on_bucket_not_found_ = retry_on_bucket_not_found;
485
493
  bootstrap_handler_ = std::move(handler);
486
494
  bootstrap_deadline_.expires_after(timeout_defaults::bootstrap_timeout);
487
495
  bootstrap_deadline_.async_wait([self = shared_from_this()](std::error_code ec) {
@@ -491,7 +499,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
491
499
  spdlog::warn("{} unable to bootstrap in time", self->log_prefix_);
492
500
  self->bootstrap_handler_(std::make_error_code(error::common_errc::unambiguous_timeout), {});
493
501
  self->bootstrap_handler_ = nullptr;
494
- self->stop();
502
+ self->stop(retry_reason::socket_closed_while_in_flight);
495
503
  });
496
504
  initiate_bootstrap();
497
505
  }
@@ -514,13 +522,17 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
514
522
  });
515
523
  return;
516
524
  }
517
- std::string service;
518
- std::tie(bootstrap_hostname_, service) = origin_.next_address();
519
- log_prefix_ = fmt::format(
520
- "[{}/{}/{}/{}] <{}:{}>", stream_->log_prefix(), client_id_, id_, bucket_name_.value_or("-"), bootstrap_hostname_, service);
525
+ std::tie(bootstrap_hostname_, bootstrap_port_) = origin_.next_address();
526
+ log_prefix_ = fmt::format("[{}/{}/{}/{}] <{}:{}>",
527
+ stream_->log_prefix(),
528
+ client_id_,
529
+ id_,
530
+ bucket_name_.value_or("-"),
531
+ bootstrap_hostname_,
532
+ bootstrap_port_);
521
533
  spdlog::debug("{} attempt to establish MCBP connection", log_prefix_);
522
534
  resolver_.async_resolve(bootstrap_hostname_,
523
- service,
535
+ bootstrap_port_,
524
536
  std::bind(&mcbp_session::on_resolve, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
525
537
  }
526
538
 
@@ -529,11 +541,17 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
529
541
  return id_;
530
542
  }
531
543
 
532
- void stop()
544
+ void on_stop(std::function<void(io::retry_reason)> handler)
545
+ {
546
+ on_stop_handler_ = std::move(handler);
547
+ }
548
+
549
+ void stop(retry_reason reason)
533
550
  {
534
551
  if (stopped_) {
535
552
  return;
536
553
  }
554
+ spdlog::debug("{} stop MCBP connection, reason={}", log_prefix_, reason);
537
555
  stopped_ = true;
538
556
  bootstrap_deadline_.cancel();
539
557
  connection_deadline_.cancel();
@@ -552,9 +570,14 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
552
570
  }
553
571
  for (auto& handler : command_handlers_) {
554
572
  spdlog::debug("{} MCBP cancel operation during session close, opaque={}, ec={}", log_prefix_, handler.first, ec.message());
555
- handler.second(ec, {});
573
+ handler.second(ec, reason, {});
556
574
  }
557
575
  command_handlers_.clear();
576
+ config_listeners_.clear();
577
+ if (on_stop_handler_) {
578
+ on_stop_handler_(reason);
579
+ }
580
+ on_stop_handler_ = nullptr;
558
581
  }
559
582
 
560
583
  void write(const std::vector<uint8_t>& buf)
@@ -564,7 +587,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
564
587
  }
565
588
  std::uint32_t opaque{ 0 };
566
589
  std::memcpy(&opaque, buf.data() + 12, sizeof(opaque));
567
- spdlog::debug("{} MCBP send, opaque={}, {:n}", log_prefix_, opaque, spdlog::to_hex(buf.begin(), buf.begin() + 24));
590
+ spdlog::trace("{} MCBP send, opaque={}, {:n}", log_prefix_, opaque, spdlog::to_hex(buf.begin(), buf.begin() + 24));
568
591
  SPDLOG_TRACE("{} MCBP send, opaque={}{:a}", log_prefix_, opaque, spdlog::to_hex(data));
569
592
  std::scoped_lock lock(output_buffer_mutex_);
570
593
  output_buffer_.push_back(buf);
@@ -589,33 +612,36 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
589
612
 
590
613
  void write_and_subscribe(uint32_t opaque,
591
614
  std::vector<std::uint8_t>& data,
592
- std::function<void(std::error_code, io::mcbp_message&&)> handler)
615
+ std::function<void(std::error_code, retry_reason, io::mcbp_message&&)> handler)
593
616
  {
594
617
  if (stopped_) {
595
- spdlog::warn("{} MCBP cancel operation, while trying to write to closed session opaque={}", log_prefix_, opaque);
596
- handler(std::make_error_code(error::common_errc::request_canceled), {});
618
+ spdlog::warn("{} MCBP cancel operation, while trying to write to closed session, opaque={}", log_prefix_, opaque);
619
+ handler(std::make_error_code(error::common_errc::request_canceled), retry_reason::socket_closed_while_in_flight, {});
597
620
  return;
598
621
  }
599
622
  command_handlers_.emplace(opaque, std::move(handler));
600
623
  if (bootstrapped_ && stream_->is_open()) {
601
624
  write_and_flush(data);
602
625
  } else {
626
+ spdlog::debug("{} the stream is not ready yet, put the message into pending buffer, opaque={}", log_prefix_, opaque);
603
627
  std::scoped_lock lock(pending_buffer_mutex_);
604
628
  pending_buffer_.push_back(data);
605
629
  }
606
630
  }
607
631
 
608
- void cancel(uint32_t opaque, std::error_code ec)
632
+ [[nodiscard]] bool cancel(uint32_t opaque, std::error_code ec, retry_reason reason)
609
633
  {
610
634
  if (stopped_) {
611
- return;
635
+ return false;
612
636
  }
613
637
  auto handler = command_handlers_.find(opaque);
614
638
  if (handler != command_handlers_.end()) {
615
639
  spdlog::debug("{} MCBP cancel operation, opaque={}, ec={}", log_prefix_, opaque, ec.message());
616
- handler->second(ec, {});
640
+ handler->second(ec, reason, {});
617
641
  command_handlers_.erase(handler);
642
+ return true;
618
643
  }
644
+ return false;
619
645
  }
620
646
 
621
647
  [[nodiscard]] bool supports_feature(protocol::hello_feature feature)
@@ -654,6 +680,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
654
680
  return bootstrap_hostname_;
655
681
  }
656
682
 
683
+ [[nodiscard]] const std::string& bootstrap_port() const
684
+ {
685
+ return bootstrap_port_;
686
+ }
687
+
657
688
  [[nodiscard]] uint32_t next_opaque()
658
689
  {
659
690
  return ++opaque_;
@@ -708,7 +739,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
708
739
  return std::make_error_code(error::common_errc::internal_server_failure);
709
740
 
710
741
  case protocol::status::busy:
711
- case protocol::status::temp_failure:
742
+ case protocol::status::temporary_failure:
712
743
  case protocol::status::no_memory:
713
744
  case protocol::status::not_initialized:
714
745
  return std::make_error_code(error::common_errc::temporary_failure);
@@ -793,10 +824,26 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
793
824
  break;
794
825
  }
795
826
  // FIXME: use error map here
796
- spdlog::warn("{} unknown status code: {} (opcode={})", log_prefix_, status, opcode);
827
+ spdlog::warn("{} unknown status code: {} (opcode={})", log_prefix_, protocol::status_to_string(status), opcode);
797
828
  return std::make_error_code(error::network_errc::protocol_error);
798
829
  }
799
830
 
831
+ std::optional<error_map::error_info> decode_error_code(std::uint16_t code)
832
+ {
833
+ if (error_map_) {
834
+ auto info = error_map_->errors.find(code);
835
+ if (info != error_map_->errors.end()) {
836
+ return info->second;
837
+ }
838
+ }
839
+ return {};
840
+ }
841
+
842
+ void on_configuration_update(std::function<void(const configuration&)> handler)
843
+ {
844
+ config_listeners_.emplace_back(std::move(handler));
845
+ }
846
+
800
847
  void update_configuration(configuration&& config)
801
848
  {
802
849
  if (stopped_) {
@@ -804,12 +851,45 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
804
851
  }
805
852
  if (!config_ || config.rev > config_->rev) {
806
853
  for (auto& node : config.nodes) {
807
- if (node.this_node && node.hostname.empty()) {
808
- node.hostname = endpoint_address_;
854
+ if (node.hostname.empty()) {
855
+ node.hostname = bootstrap_hostname_;
809
856
  }
810
857
  }
811
858
  config_.emplace(config);
812
859
  spdlog::debug("{} received new configuration: {}", log_prefix_, config_.value());
860
+ for (auto& listener : config_listeners_) {
861
+ listener(*config_);
862
+ }
863
+ }
864
+ }
865
+
866
+ void handle_not_my_vbucket(io::mcbp_message&& msg)
867
+ {
868
+ if (stopped_) {
869
+ return;
870
+ }
871
+ Expects(msg.header.magic == static_cast<std::uint8_t>(protocol::magic::alt_client_response) ||
872
+ msg.header.magic == static_cast<std::uint8_t>(protocol::magic::client_response));
873
+ if (protocol::has_json_datatype(msg.header.datatype)) {
874
+ auto magic = static_cast<protocol::magic>(msg.header.magic);
875
+ uint8_t extras_size = msg.header.extlen;
876
+ uint8_t framing_extras_size = 0;
877
+ uint16_t key_size = htons(msg.header.keylen);
878
+ if (magic == protocol::magic::alt_client_response) {
879
+ framing_extras_size = static_cast<std::uint8_t>(msg.header.keylen >> 8U);
880
+ key_size = msg.header.keylen & 0xffU;
881
+ }
882
+
883
+ std::vector<uint8_t>::difference_type offset = framing_extras_size + key_size + extras_size;
884
+ if (ntohl(msg.header.bodylen) - offset > 0) {
885
+ auto config = protocol::parse_config(msg.body.begin() + offset, msg.body.end());
886
+ spdlog::debug("{} received not_my_vbucket status for {}, opaque={} with config rev={} in the payload",
887
+ log_prefix_,
888
+ static_cast<protocol::client_opcode>(msg.header.opcode),
889
+ msg.header.opaque,
890
+ config.rev);
891
+ update_configuration(std::move(config));
892
+ }
813
893
  }
814
894
  }
815
895
 
@@ -829,13 +909,18 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
829
909
  private:
830
910
  void invoke_bootstrap_handler(std::error_code ec)
831
911
  {
912
+ if (retry_bootstrap_on_bucket_not_found_ && ec == std::make_error_code(error::common_errc::bucket_not_found)) {
913
+ spdlog::debug(R"({} server returned {}, it must be transient condition, retrying)", log_prefix_, ec.message());
914
+ return initiate_bootstrap();
915
+ }
916
+
832
917
  if (!bootstrapped_ && bootstrap_handler_) {
833
918
  bootstrap_deadline_.cancel();
834
919
  bootstrap_handler_(ec, config_.value_or(configuration{}));
835
920
  bootstrap_handler_ = nullptr;
836
921
  }
837
922
  if (ec) {
838
- return stop();
923
+ return stop(retry_reason::node_not_available);
839
924
  }
840
925
  bootstrapped_ = true;
841
926
  handler_ = std::make_unique<normal_handler>(shared_from_this());
@@ -934,8 +1019,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
934
1019
  return;
935
1020
  }
936
1021
  if (ec) {
937
- spdlog::error("{} IO error while reading from the socket: {}", self->log_prefix_, ec.message());
938
- return self->stop();
1022
+ spdlog::error("{} IO error while reading from the socket: {}, pending_handlers={}",
1023
+ self->log_prefix_,
1024
+ ec.message(),
1025
+ self->command_handlers_.size());
1026
+ return self->stop(retry_reason::socket_closed_while_in_flight);
939
1027
  }
940
1028
  self->parser_.feed(self->input_buffer_.data(), self->input_buffer_.data() + ssize_t(bytes_transferred));
941
1029
 
@@ -943,7 +1031,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
943
1031
  mcbp_message msg{};
944
1032
  switch (self->parser_.next(msg)) {
945
1033
  case mcbp_parser::ok:
946
- spdlog::debug(
1034
+ spdlog::trace(
947
1035
  "{} MCBP recv, opaque={}, {:n}", self->log_prefix_, msg.header.opaque, spdlog::to_hex(msg.header_data()));
948
1036
  SPDLOG_TRACE("{} MCBP recv, opaque={}{:a}{:a}",
949
1037
  self->log_prefix_,
@@ -962,7 +1050,7 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
962
1050
  }
963
1051
  return;
964
1052
  case mcbp_parser::failure:
965
- return self->stop();
1053
+ return self->stop(retry_reason::kv_temporary_failure);
966
1054
  }
967
1055
  }
968
1056
  });
@@ -988,8 +1076,11 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
988
1076
  return;
989
1077
  }
990
1078
  if (ec) {
991
- spdlog::error("{} IO error while writing to the socket: {}", self->log_prefix_, ec.message());
992
- return self->stop();
1079
+ spdlog::error("{} IO error while writing to the socket: {}, pending_handlers={}",
1080
+ self->log_prefix_,
1081
+ ec.message(),
1082
+ self->command_handlers_.size());
1083
+ return self->stop(retry_reason::socket_closed_while_in_flight);
993
1084
  }
994
1085
  {
995
1086
  std::scoped_lock inner_lock(self->writing_buffer_mutex_);
@@ -1013,13 +1104,16 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
1013
1104
  mcbp_parser parser_;
1014
1105
  std::unique_ptr<message_handler> handler_;
1015
1106
  std::function<void(std::error_code, const configuration&)> bootstrap_handler_{};
1016
- std::map<uint32_t, std::function<void(std::error_code, io::mcbp_message&&)>> command_handlers_{};
1107
+ std::map<uint32_t, std::function<void(std::error_code, retry_reason, io::mcbp_message&&)>> command_handlers_{};
1108
+ std::vector<std::function<void(const configuration&)>> config_listeners_{};
1109
+ std::function<void(io::retry_reason)> on_stop_handler_{};
1017
1110
 
1018
1111
  bool bootstrapped_{ false };
1019
1112
  std::atomic_bool stopped_{ false };
1020
1113
  bool authenticated_{ false };
1021
1114
  bool bucket_selected_{ false };
1022
1115
  bool supports_gcccp_{ true };
1116
+ bool retry_bootstrap_on_bucket_not_found_{ false };
1023
1117
 
1024
1118
  std::atomic<std::uint32_t> opaque_{ 0 };
1025
1119
 
@@ -1031,12 +1125,13 @@ class mcbp_session : public std::enable_shared_from_this<mcbp_session>
1031
1125
  std::mutex pending_buffer_mutex_{};
1032
1126
  std::mutex writing_buffer_mutex_{};
1033
1127
  std::string bootstrap_hostname_{};
1128
+ std::string bootstrap_port_{};
1034
1129
  asio::ip::tcp::endpoint endpoint_{}; // connected endpoint
1035
1130
  std::string endpoint_address_{}; // cached string with endpoint address
1036
1131
  asio::ip::tcp::resolver::results_type endpoints_;
1037
1132
  std::vector<protocol::hello_feature> supported_features_;
1038
1133
  std::optional<configuration> config_;
1039
- std::optional<error_map> errmap_;
1134
+ std::optional<error_map> error_map_;
1040
1135
  collection_cache collection_cache_;
1041
1136
 
1042
1137
  std::atomic_bool reading_{ false };