couchbase 3.0.0.alpha.2-universal-darwin-19 → 3.0.0-universal-darwin-19

data/ext/couchbase/io/retry_action.hxx

@@ -0,0 +1,30 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <chrono>
+#include <optional>
+
+namespace couchbase::io
+{
+struct retry_action {
+    bool retry_requested{ false };
+    std::chrono::milliseconds duration{};
+};
+
+} // namespace couchbase::io

data/ext/couchbase/io/retry_context.hxx

@@ -0,0 +1,39 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <set>
+#include <chrono>
+
+#include <io/retry_reason.hxx>
+#include <io/retry_strategy.hxx>
+
+namespace couchbase::io
+{
+
+template<class RetryStrategy>
+struct retry_context
+{
+    bool idempotent;
+    int retry_attempts{ 0 };
+    std::chrono::milliseconds last_duration{ 0 };
+    std::set<retry_reason> reasons{};
+    RetryStrategy strategy{};
+};
+
+} // namespace couchbase::io

data/ext/couchbase/io/retry_orchestrator.hxx

@@ -0,0 +1,96 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <spdlog/spdlog.h>
+
+#include <io/retry_reason.hxx>
+#include <io/retry_action.hxx>
+
+namespace couchbase::io::retry_orchestrator
+{
+
+namespace priv
+{
+template<class Command>
+std::chrono::milliseconds
+cap_duration(std::chrono::milliseconds uncapped, std::shared_ptr<Command> command)
+{
+    auto theoretical_deadline = std::chrono::steady_clock::now() + uncapped;
+    auto absolute_deadline = command->deadline.expiry();
+    auto delta = std::chrono::duration_cast<std::chrono::milliseconds>(theoretical_deadline - absolute_deadline);
+    if (delta.count() > 0) {
+        auto capped = uncapped - delta;
+        if (capped.count() < 0) {
+            return uncapped; // something went wrong, return the uncapped one as a safety net
+        }
+        return capped;
+    }
+    return uncapped;
+}
+
+std::chrono::milliseconds
+controlled_backoff(int retry_attempts)
+{
+    switch (retry_attempts) {
+        case 0:
+            return std::chrono::milliseconds(1);
+        case 1:
+            return std::chrono::milliseconds(10);
+        case 2:
+            return std::chrono::milliseconds(50);
+        case 3:
+            return std::chrono::milliseconds(100);
+        case 4:
+            return std::chrono::milliseconds(500);
+        default:
+            return std::chrono::milliseconds(1000);
+    }
+}
+
+template<class Manager, class Command>
+void
+retry_with_duration(std::shared_ptr<Manager> manager,
+                    std::shared_ptr<Command> command,
+                    retry_reason reason,
+                    std::chrono::milliseconds duration)
+{
+    ++command->request.retries.retry_attempts;
+    command->request.retries.reasons.insert(reason);
+    command->request.retries.last_duration = duration;
+    manager->schedule_for_retry(command, duration);
+}
+
+} // namespace priv
+
+template<class Manager, class Command>
+void
+maybe_retry(std::shared_ptr<Manager> manager, std::shared_ptr<Command> command, retry_reason reason, std::error_code ec)
+{
+    if (always_retry(reason)) {
+        return priv::retry_with_duration(manager, command, reason, priv::controlled_backoff(command->request.retries.retry_attempts));
+    }
+
+    retry_action action = command->request.retries.strategy.should_retry(command->request, reason);
+    if (action.retry_requested) {
+        return priv::retry_with_duration(manager, command, reason, priv::cap_duration(action.duration, command));
+    }
+    return command->invoke_handler(ec);
+}
+
+} // namespace couchbase::io::retry_orchestrator

data/ext/couchbase/io/retry_reason.hxx

@@ -0,0 +1,235 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <spdlog/fmt/fmt.h>
+
+namespace couchbase::io
+{
+enum class retry_reason {
+    /**
+     * default value, e.g. when we don't need to retry
+     */
+    do_not_retry,
+
+    /**
+     * All unexpected/unknown retry errors must not be retried to avoid accidental data loss and non-deterministic behavior.
+     */
+    unknown,
+
+    /**
+     * The socket is not available into which the operation should’ve been written.
+     */
+    socket_not_available,
+
+    /**
+     * The service on a node (i.e. kv, query) is not available.
+     */
+    service_not_available,
+
+    /**
+     * The node where the operation is supposed to be dispatched to is not available.
+     */
+    node_not_available,
+
+    /**
+     * A not my vbucket response has been received.
+     */
+    kv_not_my_vbucket,
+
+    /**
+     * A KV response has been received which signals an outdated collection.
+     */
+    kv_collection_outdated,
+
+    /**
+     * An unknown response was returned and the consulted KV error map indicated a retry.
+     */
+    kv_error_map_retry_indicated,
+
+    kv_locked,
+
+    kv_temporary_failure,
+
+    kv_sync_write_in_progress,
+
+    kv_sync_write_re_commit_in_progress,
+
+    service_response_code_indicated,
+
+    /**
+     * While an operation was in-flight, the underlying socket has been closed.
+     */
+    socket_closed_while_in_flight,
+
+    /**
+     * The circuit breaker is open for the given socket/endpoint and as a result the operation is not sent into it.
+     */
+    circuit_breaker_open,
+
+    query_prepared_statement_failure,
+
+    query_index_not_found,
+
+    analytics_temporary_failure,
+
+    search_too_many_requests,
+
+    views_temporary_failure,
+
+    views_no_active_partition,
+};
+
+constexpr inline bool
+allows_non_idempotent_retry(retry_reason reason)
+{
+    switch (reason) {
+        case retry_reason::socket_not_available:
+        case retry_reason::service_not_available:
+        case retry_reason::node_not_available:
+        case retry_reason::kv_not_my_vbucket:
+        case retry_reason::kv_collection_outdated:
+        case retry_reason::kv_error_map_retry_indicated:
+        case retry_reason::kv_locked:
+        case retry_reason::kv_temporary_failure:
+        case retry_reason::kv_sync_write_in_progress:
+        case retry_reason::kv_sync_write_re_commit_in_progress:
+        case retry_reason::service_response_code_indicated:
+        case retry_reason::circuit_breaker_open:
+        case retry_reason::query_prepared_statement_failure:
+        case retry_reason::query_index_not_found:
+        case retry_reason::analytics_temporary_failure:
+        case retry_reason::search_too_many_requests:
+        case retry_reason::views_temporary_failure:
+        case retry_reason::views_no_active_partition:
+            return true;
+        case retry_reason::do_not_retry:
+        case retry_reason::socket_closed_while_in_flight:
+        case retry_reason::unknown:
+            return false;
+    }
+    return false;
+}
+
+constexpr inline bool
+always_retry(retry_reason reason)
+{
+    switch (reason) {
+        case retry_reason::kv_not_my_vbucket:
+        case retry_reason::kv_collection_outdated:
+        case retry_reason::views_no_active_partition:
+            return true;
+        case retry_reason::do_not_retry:
+        case retry_reason::socket_not_available:
+        case retry_reason::service_not_available:
+        case retry_reason::node_not_available:
+        case retry_reason::kv_error_map_retry_indicated:
+        case retry_reason::kv_locked:
+        case retry_reason::kv_temporary_failure:
+        case retry_reason::kv_sync_write_in_progress:
+        case retry_reason::kv_sync_write_re_commit_in_progress:
+        case retry_reason::service_response_code_indicated:
+        case retry_reason::socket_closed_while_in_flight:
+        case retry_reason::circuit_breaker_open:
+        case retry_reason::query_prepared_statement_failure:
+        case retry_reason::query_index_not_found:
+        case retry_reason::analytics_temporary_failure:
+        case retry_reason::search_too_many_requests:
+        case retry_reason::views_temporary_failure:
+        case retry_reason::unknown:
+            return false;
+    }
+    return false;
+}
+
+} // namespace couchbase::io
+
+template<>
+struct fmt::formatter<couchbase::io::retry_reason> : formatter<string_view> {
+    template<typename FormatContext>
+    auto format(couchbase::io::retry_reason reason, FormatContext& ctx)
+    {
+        string_view name = "unknown";
+        switch (reason) {
+            case couchbase::io::retry_reason::do_not_retry:
+                name = "do_not_retry";
+                break;
+            case couchbase::io::retry_reason::unknown:
+                name = "unknown";
+                break;
+            case couchbase::io::retry_reason::socket_not_available:
+                name = "socket_not_available";
+                break;
+            case couchbase::io::retry_reason::service_not_available:
+                name = "service_not_available";
+                break;
+            case couchbase::io::retry_reason::node_not_available:
+                name = "node_not_available";
+                break;
+            case couchbase::io::retry_reason::kv_not_my_vbucket:
+                name = "kv_not_my_vbucket";
+                break;
+            case couchbase::io::retry_reason::kv_collection_outdated:
+                name = "kv_collection_outdated";
+                break;
+            case couchbase::io::retry_reason::kv_error_map_retry_indicated:
+                name = "kv_error_map_retry_indicated";
+                break;
+            case couchbase::io::retry_reason::kv_locked:
+                name = "kv_locked";
+                break;
+            case couchbase::io::retry_reason::kv_temporary_failure:
+                name = "kv_temporary_failure";
+                break;
+            case couchbase::io::retry_reason::kv_sync_write_in_progress:
+                name = "kv_sync_write_in_progress";
+                break;
+            case couchbase::io::retry_reason::kv_sync_write_re_commit_in_progress:
+                name = "kv_sync_write_re_commit_in_progress";
+                break;
+            case couchbase::io::retry_reason::service_response_code_indicated:
+                name = "service_response_code_indicated";
+                break;
+            case couchbase::io::retry_reason::socket_closed_while_in_flight:
+                name = "socket_closed_while_in_flight";
+                break;
+            case couchbase::io::retry_reason::circuit_breaker_open:
+                name = "circuit_breaker_open";
+                break;
+            case couchbase::io::retry_reason::query_prepared_statement_failure:
+                name = "query_prepared_statement_failure";
+                break;
+            case couchbase::io::retry_reason::query_index_not_found:
+                name = "query_index_not_found";
+                break;
+            case couchbase::io::retry_reason::analytics_temporary_failure:
+                name = "analytics_temporary_failure";
+                break;
+            case couchbase::io::retry_reason::search_too_many_requests:
+                name = "search_too_many_requests";
+                break;
+            case couchbase::io::retry_reason::views_temporary_failure:
+                name = "views_temporary_failure";
+                break;
+            case couchbase::io::retry_reason::views_no_active_partition:
+                name = "views_no_active_partition";
+                break;
+        }
+        return formatter<string_view>::format(name, ctx);
+    }
+};

data/ext/couchbase/io/retry_strategy.hxx

@@ -0,0 +1,156 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2020 Couchbase, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+#pragma once
+
+#include <gsl/gsl_assert>
+
+#include <io/retry_reason.hxx>
+#include <io/retry_action.hxx>
+
+namespace couchbase::io::retry_strategy
+{
+namespace backoff
+{
+struct backoff_delay {
+    std::chrono::milliseconds delay;
+    std::chrono::milliseconds min_delay;
+    std::chrono::milliseconds max_delay;
+};
+
+class fixed
+{
+  private:
+    backoff_delay delay_;
+
+  public:
+    explicit fixed(std::chrono::milliseconds delay)
+      : delay_{ delay, delay, delay }
+    {
+    }
+
+    template<class Request>
+    backoff_delay operator()(Request&)
+    {
+        return delay_;
+    }
+};
+
+class exponential
+{
+  public:
+    /**
+     * Backoff function with exponential backoff delay. Retries are performed after a backoff interval of
+     *
+     *   first_backoff * (factor^n)
+     *
+     * where n is the iteration. If max_backoff is not zero, the maximum backoff applied will be limited to max_backoff.
+     *
+     * If based_on_previous_value is true, backoff will be calculated using
+     *
+     *   prev_backoff * factor
+     *
+     * When backoffs are combined with jitter, this value will be different from the actual exponential value for the iteration.
+     *
+     * @param first_backoff First backoff duration
+     * @param factor The multiplicand for calculating backoff
+     * @param max_backoff Maximum backoff duration
+     * @param based_on_previous_value If true, calculation is based on previous value which may be a backoff with jitter applied
+     */
+    exponential(std::chrono::milliseconds first_backoff,
+                int factor,
+                std::chrono::milliseconds max_backoff = {},
+                bool based_on_previous_value = false)
+    {
+        Expects(first_backoff.count() > 0);
+        if (max_backoff.count() <= 0) {
+            max_backoff = std::chrono::milliseconds{ std::numeric_limits<std::chrono::milliseconds::rep>::max() };
+        }
+        Expects(max_backoff > first_backoff);
+        first_backoff_ = first_backoff;
+        max_backoff_ = max_backoff;
+        factor_ = factor;
+        based_on_previous_value_ = based_on_previous_value;
+    }
+
+    template<class Request>
+    backoff_delay operator()(Request& request)
+    {
+        std::chrono::milliseconds previous_backoff = request.retries.last_duration;
+        std::chrono::milliseconds next_backoff;
+
+        if (based_on_previous_value_) {
+            if (previous_backoff >= max_backoff_) {
+                next_backoff = max_backoff_;
+            } else {
+                next_backoff = previous_backoff * factor_;
+            }
+            if (next_backoff < first_backoff_) {
+                next_backoff = first_backoff_;
+            }
+        } else {
+            if (previous_backoff >= max_backoff_) {
+                next_backoff = max_backoff_;
+            } else {
+                next_backoff = first_backoff_ * static_cast<int>(std::pow(factor_, request.retries.retry_attempts - 1));
+            }
+        }
+        return { next_backoff, first_backoff_, max_backoff_ };
+    }
+
+  private:
+    std::chrono::milliseconds first_backoff_;
+    std::chrono::milliseconds max_backoff_;
+    int factor_;
+    bool based_on_previous_value_;
+};
+
+} // namespace backoff
+
+class best_effort
+{
+  public:
+    best_effort()
+      : backoff_(std::chrono::milliseconds(1), 2, std::chrono::milliseconds(500))
+    {
+    }
+
+    template<class Request>
+    retry_action should_retry(Request& request, retry_reason reason)
+    {
+        if (request.retries.idempotent || allows_non_idempotent_retry(reason)) {
+            backoff::backoff_delay delay = backoff_(request);
+            return { true, delay.delay };
+        }
+        return { false };
+    }
+
+  private:
+    backoff::exponential backoff_;
+};
+
+class fail_fast
+{
+  public:
+    template<class Request>
+    retry_action should_retry(Request&, retry_reason)
+    {
+        return { false };
+    }
+};
+
+} // namespace couchbase::io::retry_strategy