grpc 1.28.0 → 1.37.0

data/src/core/ext/filters/client_channel/subchannel_pool_interface.cc

@@ -44,13 +44,27 @@ SubchannelKey::SubchannelKey(const SubchannelKey& other) {
 }
 
 SubchannelKey& SubchannelKey::operator=(const SubchannelKey& other) {
+  if (&other == this) {
+    return *this;
+  }
   grpc_channel_args_destroy(const_cast<grpc_channel_args*>(args_));
   Init(other.args_, grpc_channel_args_copy);
   return *this;
 }
 
-int SubchannelKey::Cmp(const SubchannelKey& other) const {
-  return grpc_channel_args_compare(args_, other.args_);
+SubchannelKey::SubchannelKey(SubchannelKey&& other) noexcept {
+  args_ = other.args_;
+  other.args_ = nullptr;
+}
+
+SubchannelKey& SubchannelKey::operator=(SubchannelKey&& other) noexcept {
+  args_ = other.args_;
+  other.args_ = nullptr;
+  return *this;
+}
+
+bool SubchannelKey::operator<(const SubchannelKey& other) const {
+  return grpc_channel_args_compare(args_, other.args_) < 0;
 }
 
 void SubchannelKey::Init(

data/src/core/ext/filters/client_channel/subchannel_pool_interface.h

@@ -23,6 +23,7 @@
 
 #include "src/core/lib/avl/avl.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 
 namespace grpc_core {
@@ -40,11 +41,11 @@ class SubchannelKey {
   // Copyable.
   SubchannelKey(const SubchannelKey& other);
   SubchannelKey& operator=(const SubchannelKey& other);
-  // Not movable.
-  SubchannelKey(SubchannelKey&&) = delete;
-  SubchannelKey& operator=(SubchannelKey&&) = delete;
+  // Movable
+  SubchannelKey(SubchannelKey&&) noexcept;
+  SubchannelKey& operator=(SubchannelKey&&) noexcept;
 
-  int Cmp(const SubchannelKey& other) const;
+  bool operator<(const SubchannelKey& other) const;
 
  private:
   // Initializes the subchannel key with the given \a args and the function to
@@ -62,21 +63,26 @@ class SubchannelKey {
 // shut down safely. See https://github.com/grpc/grpc/issues/12560.
 class SubchannelPoolInterface : public RefCounted<SubchannelPoolInterface> {
  public:
-  SubchannelPoolInterface() : RefCounted(&grpc_subchannel_pool_trace) {}
-  virtual ~SubchannelPoolInterface() {}
+  SubchannelPoolInterface()
+      : RefCounted(GRPC_TRACE_FLAG_ENABLED(grpc_subchannel_pool_trace)
+                       ? "SubchannelPoolInterface"
+                       : nullptr) {}
+  ~SubchannelPoolInterface() override {}
 
   // Registers a subchannel against a key. Returns the subchannel registered
   // with \a key, which may be different from \a constructed because we reuse
   // (instead of update) any existing subchannel already registered with \a key.
-  virtual Subchannel* RegisterSubchannel(SubchannelKey* key,
-                                         Subchannel* constructed) = 0;
+  virtual RefCountedPtr<Subchannel> RegisterSubchannel(
+      const SubchannelKey& key, RefCountedPtr<Subchannel> constructed) = 0;
 
   // Removes the registered subchannel found by \a key.
-  virtual void UnregisterSubchannel(SubchannelKey* key) = 0;
+  virtual void UnregisterSubchannel(const SubchannelKey& key,
+                                    Subchannel* subchannel) = 0;
 
   // Finds the subchannel registered for the given subchannel key. Returns NULL
   // if no such channel exists. Thread-safe.
-  virtual Subchannel* FindSubchannel(SubchannelKey* key) = 0;
+  virtual RefCountedPtr<Subchannel> FindSubchannel(
+      const SubchannelKey& key) = 0;
 
   // Creates a channel arg from \a subchannel pool.
   static grpc_arg CreateChannelArg(SubchannelPoolInterface* subchannel_pool);

data/src/core/ext/filters/client_idle/client_idle_filter.cc

@@ -187,7 +187,7 @@ void ChannelData::StartTransportOp(grpc_channel_element* elem,
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   // Catch the disconnect_with_error transport op.
   if (op->disconnect_with_error != nullptr) {
-    // IncreaseCallCount() introduces a dummy call and prevent the timer from
+    // IncreaseCallCount() introduces a phony call and prevent the timer from
     // being reset by other threads.
     chand->IncreaseCallCount();
     // If the timer has been set, cancel the timer.

data/src/core/ext/filters/deadline/deadline_filter.cc

@@ -32,53 +32,82 @@
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/channel_init.h"
 
-//
-// grpc_deadline_state
-//
+namespace grpc_core {
+
+// A fire-and-forget class representing a pending deadline timer.
+// Allocated on the call arena.
+class TimerState {
+ public:
+  TimerState(grpc_call_element* elem, grpc_millis deadline) : elem_(elem) {
+    grpc_deadline_state* deadline_state =
+        static_cast<grpc_deadline_state*>(elem_->call_data);
+    GRPC_CALL_STACK_REF(deadline_state->call_stack, "DeadlineTimerState");
+    GRPC_CLOSURE_INIT(&closure_, TimerCallback, this, nullptr);
+    grpc_timer_init(&timer_, deadline, &closure_);
+  }
 
-// The on_complete callback used when sending a cancel_error batch down the
-// filter stack.  Yields the call combiner when the batch returns.
-static void yield_call_combiner(void* arg, grpc_error* /*ignored*/) {
-  grpc_deadline_state* deadline_state = static_cast<grpc_deadline_state*>(arg);
-  GRPC_CALL_COMBINER_STOP(deadline_state->call_combiner,
-                          "got on_complete from cancel_stream batch");
-  GRPC_CALL_STACK_UNREF(deadline_state->call_stack, "deadline_timer");
-}
+  void Cancel() { grpc_timer_cancel(&timer_); }
+
+ private:
+  // The on_complete callback used when sending a cancel_error batch down the
+  // filter stack.  Yields the call combiner when the batch returns.
+  static void YieldCallCombiner(void* arg, grpc_error* /*ignored*/) {
+    TimerState* self = static_cast<TimerState*>(arg);
+    grpc_deadline_state* deadline_state =
+        static_cast<grpc_deadline_state*>(self->elem_->call_data);
+    GRPC_CALL_COMBINER_STOP(deadline_state->call_combiner,
+                            "got on_complete from cancel_stream batch");
+    GRPC_CALL_STACK_UNREF(deadline_state->call_stack, "DeadlineTimerState");
+  }
 
-// This is called via the call combiner, so access to deadline_state is
-// synchronized.
-static void send_cancel_op_in_call_combiner(void* arg, grpc_error* error) {
-  grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
-  grpc_deadline_state* deadline_state =
-      static_cast<grpc_deadline_state*>(elem->call_data);
-  grpc_transport_stream_op_batch* batch = grpc_make_transport_stream_op(
-      GRPC_CLOSURE_INIT(&deadline_state->timer_callback, yield_call_combiner,
-                        deadline_state, grpc_schedule_on_exec_ctx));
-  batch->cancel_stream = true;
-  batch->payload->cancel_stream.cancel_error = GRPC_ERROR_REF(error);
-  elem->filter->start_transport_stream_op_batch(elem, batch);
-}
+  // This is called via the call combiner, so access to deadline_state is
+  // synchronized.
+  static void SendCancelOpInCallCombiner(void* arg, grpc_error* error) {
+    TimerState* self = static_cast<TimerState*>(arg);
+    grpc_transport_stream_op_batch* batch = grpc_make_transport_stream_op(
+        GRPC_CLOSURE_INIT(&self->closure_, YieldCallCombiner, self, nullptr));
+    batch->cancel_stream = true;
+    batch->payload->cancel_stream.cancel_error = GRPC_ERROR_REF(error);
+    self->elem_->filter->start_transport_stream_op_batch(self->elem_, batch);
+  }
 
-// Timer callback.
-static void timer_callback(void* arg, grpc_error* error) {
-  grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
-  grpc_deadline_state* deadline_state =
-      static_cast<grpc_deadline_state*>(elem->call_data);
-  if (error != GRPC_ERROR_CANCELLED) {
-    error = grpc_error_set_int(
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Deadline Exceeded"),
-        GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_DEADLINE_EXCEEDED);
-    deadline_state->call_combiner->Cancel(GRPC_ERROR_REF(error));
-    GRPC_CLOSURE_INIT(&deadline_state->timer_callback,
-                      send_cancel_op_in_call_combiner, elem,
-                      grpc_schedule_on_exec_ctx);
-    GRPC_CALL_COMBINER_START(deadline_state->call_combiner,
-                             &deadline_state->timer_callback, error,
-                             "deadline exceeded -- sending cancel_stream op");
-  } else {
-    GRPC_CALL_STACK_UNREF(deadline_state->call_stack, "deadline_timer");
+  // Timer callback.
+  static void TimerCallback(void* arg, grpc_error* error) {
+    TimerState* self = static_cast<TimerState*>(arg);
+    grpc_deadline_state* deadline_state =
+        static_cast<grpc_deadline_state*>(self->elem_->call_data);
+    if (error != GRPC_ERROR_CANCELLED) {
+      error = grpc_error_set_int(
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Deadline Exceeded"),
+          GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_DEADLINE_EXCEEDED);
+      deadline_state->call_combiner->Cancel(GRPC_ERROR_REF(error));
+      GRPC_CLOSURE_INIT(&self->closure_, SendCancelOpInCallCombiner, self,
+                        nullptr);
+      GRPC_CALL_COMBINER_START(deadline_state->call_combiner, &self->closure_,
+                               error,
+                               "deadline exceeded -- sending cancel_stream op");
+    } else {
+      GRPC_CALL_STACK_UNREF(deadline_state->call_stack, "DeadlineTimerState");
+    }
   }
-}
+
+  // NOTE: This object's dtor is never called, so do not add any data
+  // members that require destruction!
+  // TODO(roth): We should ideally call this object's dtor somewhere,
+  // but that would require adding more synchronization, because we'd
+  // need to call the dtor only after both (a) the timer callback
+  // finishes and (b) the filter sees the call completion and attempts
+  // to cancel the timer.
+  grpc_call_element* elem_;
+  grpc_timer timer_;
+  grpc_closure closure_;
+};
+
+}  // namespace grpc_core
+
+//
+// grpc_deadline_state
+//
 
 // Starts the deadline timer.
 // This is called via the call combiner, so access to deadline_state is
@@ -90,41 +119,18 @@ static void start_timer_if_needed(grpc_call_element* elem,
   }
   grpc_deadline_state* deadline_state =
       static_cast<grpc_deadline_state*>(elem->call_data);
-  grpc_closure* closure = nullptr;
-  switch (deadline_state->timer_state) {
-    case GRPC_DEADLINE_STATE_PENDING:
-      // Note: We do not start the timer if there is already a timer
-      return;
-    case GRPC_DEADLINE_STATE_FINISHED:
-      deadline_state->timer_state = GRPC_DEADLINE_STATE_PENDING;
-      // If we've already created and destroyed a timer, we always create a
-      // new closure: we have no other guarantee that the inlined closure is
-      // not in use (it may hold a pending call to timer_callback)
-      closure =
-          GRPC_CLOSURE_CREATE(timer_callback, elem, grpc_schedule_on_exec_ctx);
-      break;
-    case GRPC_DEADLINE_STATE_INITIAL:
-      deadline_state->timer_state = GRPC_DEADLINE_STATE_PENDING;
-      closure =
-          GRPC_CLOSURE_INIT(&deadline_state->timer_callback, timer_callback,
-                            elem, grpc_schedule_on_exec_ctx);
-      break;
-  }
-  GPR_ASSERT(closure != nullptr);
-  GRPC_CALL_STACK_REF(deadline_state->call_stack, "deadline_timer");
-  grpc_timer_init(&deadline_state->timer, deadline, closure);
+  GPR_ASSERT(deadline_state->timer_state == nullptr);
+  deadline_state->timer_state =
+      deadline_state->arena->New<grpc_core::TimerState>(elem, deadline);
 }
 
 // Cancels the deadline timer.
 // This is called via the call combiner, so access to deadline_state is
 // synchronized.
 static void cancel_timer_if_needed(grpc_deadline_state* deadline_state) {
-  if (deadline_state->timer_state == GRPC_DEADLINE_STATE_PENDING) {
-    deadline_state->timer_state = GRPC_DEADLINE_STATE_FINISHED;
-    grpc_timer_cancel(&deadline_state->timer);
-  } else {
-    // timer was either in STATE_INITIAL (nothing to cancel)
-    // OR in STATE_FINISHED (again nothing to cancel)
+  if (deadline_state->timer_state != nullptr) {
+    deadline_state->timer_state->Cancel();
+    deadline_state->timer_state = nullptr;
   }
 }
 
@@ -182,10 +188,11 @@ static void start_timer_after_init(void* arg, grpc_error* error) {
 }
 
 grpc_deadline_state::grpc_deadline_state(grpc_call_element* elem,
-                                         grpc_call_stack* call_stack,
-                                         grpc_core::CallCombiner* call_combiner,
+                                         const grpc_call_element_args& args,
                                          grpc_millis deadline)
-    : call_stack(call_stack), call_combiner(call_combiner) {
+    : call_stack(args.call_stack),
+      call_combiner(args.call_combiner),
+      arena(args.arena) {
   // Deadline will always be infinite on servers, so the timer will only be
   // set on clients with a finite deadline.
   if (deadline != GRPC_MILLIS_INF_FUTURE) {
@@ -263,8 +270,7 @@ typedef struct server_call_data {
 // Constructor for call_data.  Used for both client and server filters.
 static grpc_error* deadline_init_call_elem(grpc_call_element* elem,
                                            const grpc_call_element_args* args) {
-  new (elem->call_data) grpc_deadline_state(
-      elem, args->call_stack, args->call_combiner, args->deadline);
+  new (elem->call_data) grpc_deadline_state(elem, *args, args->deadline);
   return GRPC_ERROR_NONE;
 }
 
@@ -377,10 +383,12 @@ static bool maybe_add_deadline_filter(grpc_channel_stack_builder* builder,
 void grpc_deadline_filter_init(void) {
   grpc_channel_init_register_stage(
       GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY,
-      maybe_add_deadline_filter, (void*)&grpc_client_deadline_filter);
+      maybe_add_deadline_filter,
+      const_cast<grpc_channel_filter*>(&grpc_client_deadline_filter));
   grpc_channel_init_register_stage(
       GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY,
-      maybe_add_deadline_filter, (void*)&grpc_server_deadline_filter);
+      maybe_add_deadline_filter,
+      const_cast<grpc_channel_filter*>(&grpc_server_deadline_filter));
 }
 
 void grpc_deadline_filter_shutdown(void) {}

data/src/core/ext/filters/deadline/deadline_filter.h

@@ -22,26 +22,22 @@
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/iomgr/timer.h"
 
-enum grpc_deadline_timer_state {
-  GRPC_DEADLINE_STATE_INITIAL,
-  GRPC_DEADLINE_STATE_PENDING,
-  GRPC_DEADLINE_STATE_FINISHED
-};
+namespace grpc_core {
+class TimerState;
+}  // namespace grpc_core
 
 // State used for filters that enforce call deadlines.
 // Must be the first field in the filter's call_data.
 struct grpc_deadline_state {
-  grpc_deadline_state(grpc_call_element* elem, grpc_call_stack* call_stack,
-                      grpc_core::CallCombiner* call_combiner,
-                      grpc_millis deadline);
+  grpc_deadline_state(grpc_call_element* elem,
+                      const grpc_call_element_args& args, grpc_millis deadline);
   ~grpc_deadline_state();
 
   // We take a reference to the call stack for the timer callback.
   grpc_call_stack* call_stack;
   grpc_core::CallCombiner* call_combiner;
-  grpc_deadline_timer_state timer_state = GRPC_DEADLINE_STATE_INITIAL;
-  grpc_timer timer;
-  grpc_closure timer_callback;
+  grpc_core::Arena* arena;
+  grpc_core::TimerState* timer_state = nullptr;
   // Closure to invoke when we receive trailing metadata.
   // We use this to cancel the timer.
   grpc_closure recv_trailing_metadata_ready;

data/src/core/ext/filters/fault_injection/fault_injection_filter.cc

@@ -0,0 +1,495 @@
+//
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/ext/filters/fault_injection/fault_injection_filter.h"
+
+#include "absl/strings/numbers.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/ext/filters/client_channel/service_config.h"
+#include "src/core/ext/filters/client_channel/service_config_call_data.h"
+#include "src/core/ext/filters/fault_injection/service_config_parser.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/channel/status_util.h"
+#include "src/core/lib/gprpp/atomic.h"
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/timer.h"
+#include "src/core/lib/transport/status_conversion.h"
+
+namespace grpc_core {
+
+TraceFlag grpc_fault_injection_filter_trace(false, "fault_injection_filter");
+
+namespace {
+
+Atomic<uint32_t> g_active_faults{0};
+static_assert(
+    std::is_trivially_destructible<Atomic<uint32_t>>::value,
+    "the active fault counter needs to have a trivially destructible type");
+
+inline int GetLinkedMetadatumValueInt(grpc_linked_mdelem* md) {
+  int res;
+  if (absl::SimpleAtoi(StringViewFromSlice(GRPC_MDVALUE(md->md)), &res)) {
+    return res;
+  } else {
+    return -1;
+  }
+}
+
+inline uint32_t GetLinkedMetadatumValueUnsignedInt(grpc_linked_mdelem* md) {
+  uint32_t res;
+  if (absl::SimpleAtoi(StringViewFromSlice(GRPC_MDVALUE(md->md)), &res)) {
+    return res;
+  } else {
+    return -1;
+  }
+}
+
+inline int64_t GetLinkedMetadatumValueInt64(grpc_linked_mdelem* md) {
+  int64_t res;
+  if (absl::SimpleAtoi(StringViewFromSlice(GRPC_MDVALUE(md->md)), &res)) {
+    return res;
+  } else {
+    return -1;
+  }
+}
+
+inline bool UnderFraction(const uint32_t numerator,
+                          const uint32_t denominator) {
+  if (numerator <= 0) return false;
+  if (numerator >= denominator) return true;
+  // Generate a random number in [0, denominator).
+  const uint32_t random_number = rand() % denominator;
+  return random_number < numerator;
+}
+
+class ChannelData {
+ public:
+  static grpc_error* Init(grpc_channel_element* elem,
+                          grpc_channel_element_args* args);
+  static void Destroy(grpc_channel_element* elem);
+
+  int index() const { return index_; }
+
+ private:
+  ChannelData(grpc_channel_element* elem, grpc_channel_element_args* args);
+  ~ChannelData() = default;
+
+  // The relative index of instances of the same filter.
+  int index_;
+};
+
+class CallData {
+ public:
+  static grpc_error* Init(grpc_call_element* elem,
+                          const grpc_call_element_args* args);
+
+  static void Destroy(grpc_call_element* elem,
+                      const grpc_call_final_info* /*final_info*/,
+                      grpc_closure* /*then_schedule_closure*/);
+
+  static void StartTransportStreamOpBatch(
+      grpc_call_element* elem, grpc_transport_stream_op_batch* batch);
+
+ private:
+  class ResumeBatchCanceller;
+
+  CallData(grpc_call_element* elem, const grpc_call_element_args* args);
+  ~CallData();
+
+  void DecideWhetherToInjectFaults(grpc_metadata_batch* initial_metadata);
+
+  // Checks if current active faults exceed the allowed max faults.
+  bool HaveActiveFaultsQuota(bool increment);
+
+  // Returns true if this RPC needs to be delayed. If so, this call will be
+  // counted as an active fault.
+  bool MaybeDelay();
+
+  // Returns the aborted RPC status if this RPC needs to be aborted. If so,
+  // this call will be counted as an active fault. Otherwise, it returns
+  // GRPC_ERROR_NONE.
+  // If this call is already been delay injected, skip the active faults
+  // quota check.
+  grpc_error* MaybeAbort();
+
+  // Delays the stream operations batch.
+  void DelayBatch(grpc_call_element* elem,
+                  grpc_transport_stream_op_batch* batch);
+
+  // Cancels the delay timer.
+  void CancelDelayTimer() { grpc_timer_cancel(&delay_timer_); }
+
+  // Finishes the fault injection, should only be called once.
+  void FaultInjectionFinished() {
+    g_active_faults.FetchSub(1, MemoryOrder::RELAXED);
+  }
+
+  // This is a callback that will be invoked after the delay timer is up.
+  static void ResumeBatch(void* arg, grpc_error* error);
+
+  // This is a callback invoked upon completion of recv_trailing_metadata.
+  // Injects the abort_error_ to the recv_trailing_metadata batch if needed.
+  static void HijackedRecvTrailingMetadataReady(void* arg, grpc_error*);
+
+  // Used to track the policy structs that needs to be destroyed in dtor.
+  bool fi_policy_owned_ = false;
+  const FaultInjectionMethodParsedConfig::FaultInjectionPolicy* fi_policy_;
+  grpc_call_stack* owning_call_;
+  Arena* arena_;
+  CallCombiner* call_combiner_;
+
+  // Indicates whether we are doing a delay and/or an abort for this call.
+  bool delay_request_ = false;
+  bool abort_request_ = false;
+
+  // Delay states
+  grpc_timer delay_timer_ ABSL_GUARDED_BY(delay_mu_);
+  ResumeBatchCanceller* resume_batch_canceller_ ABSL_GUARDED_BY(delay_mu_);
+  grpc_transport_stream_op_batch* delayed_batch_ ABSL_GUARDED_BY(delay_mu_);
+  // Abort states
+  grpc_error* abort_error_ = GRPC_ERROR_NONE;
+  grpc_closure recv_trailing_metadata_ready_;
+  grpc_closure* original_recv_trailing_metadata_ready_;
+  // Protects the asynchronous delay, resume, and cancellation.
+  Mutex delay_mu_;
+};
+
+// ChannelData
+
+grpc_error* ChannelData::Init(grpc_channel_element* elem,
+                              grpc_channel_element_args* args) {
+  GPR_ASSERT(elem->filter == &FaultInjectionFilterVtable);
+  new (elem->channel_data) ChannelData(elem, args);
+  return GRPC_ERROR_NONE;
+}
+
+void ChannelData::Destroy(grpc_channel_element* elem) {
+  auto* chand = static_cast<ChannelData*>(elem->channel_data);
+  chand->~ChannelData();
+}
+
+ChannelData::ChannelData(grpc_channel_element* elem,
+                         grpc_channel_element_args* args)
+    : index_(grpc_channel_stack_filter_instance_number(args->channel_stack,
+                                                       elem)) {}
+
+// CallData::ResumeBatchCanceller
+
+class CallData::ResumeBatchCanceller {
+ public:
+  explicit ResumeBatchCanceller(grpc_call_element* elem) : elem_(elem) {
+    auto* calld = static_cast<CallData*>(elem->call_data);
+    GRPC_CALL_STACK_REF(calld->owning_call_, "ResumeBatchCanceller");
+    GRPC_CLOSURE_INIT(&closure_, &Cancel, this, grpc_schedule_on_exec_ctx);
+    calld->call_combiner_->SetNotifyOnCancel(&closure_);
+  }
+
+ private:
+  static void Cancel(void* arg, grpc_error* error) {
+    auto* self = static_cast<ResumeBatchCanceller*>(arg);
+    auto* chand = static_cast<ChannelData*>(self->elem_->channel_data);
+    auto* calld = static_cast<CallData*>(self->elem_->call_data);
+    MutexLock lock(&calld->delay_mu_);
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_fault_injection_filter_trace)) {
+      gpr_log(GPR_INFO,
+              "chand=%p calld=%p: cancelling schdueled pick: "
+              "error=%s self=%p calld->resume_batch_canceller_=%p",
+              chand, calld, grpc_error_string(error), self,
+              calld->resume_batch_canceller_);
+    }
+    if (error != GRPC_ERROR_NONE && calld->resume_batch_canceller_ == self) {
+      // Cancel the delayed pick.
+      calld->CancelDelayTimer();
+      calld->FaultInjectionFinished();
+      // Fail pending batches on the call.
+      grpc_transport_stream_op_batch_finish_with_failure(
+          calld->delayed_batch_, GRPC_ERROR_REF(error), calld->call_combiner_);
+    }
+    GRPC_CALL_STACK_UNREF(calld->owning_call_, "ResumeBatchCanceller");
+    delete self;
+  }
+
+  grpc_call_element* elem_;
+  grpc_closure closure_;
+};
+
+// CallData
+
+grpc_error* CallData::Init(grpc_call_element* elem,
+                           const grpc_call_element_args* args) {
+  auto* calld = new (elem->call_data) CallData(elem, args);
+  if (calld->fi_policy_ == nullptr) {
+    return GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "failed to find fault injection policy");
+  }
+  return GRPC_ERROR_NONE;
+}
+
+void CallData::Destroy(grpc_call_element* elem,
+                       const grpc_call_final_info* /*final_info*/,
+                       grpc_closure* /*then_schedule_closure*/) {
+  auto* calld = static_cast<CallData*>(elem->call_data);
+  calld->~CallData();
+}
+
+void CallData::StartTransportStreamOpBatch(
+    grpc_call_element* elem, grpc_transport_stream_op_batch* batch) {
+  auto* calld = static_cast<CallData*>(elem->call_data);
+  // There should only be one send_initial_metdata op, and fault injection also
+  // only need to be enforced once.
+  if (batch->send_initial_metadata) {
+    calld->DecideWhetherToInjectFaults(
+        batch->payload->send_initial_metadata.send_initial_metadata);
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_fault_injection_filter_trace)) {
+      gpr_log(GPR_INFO,
+              "chand=%p calld=%p: Fault injection triggered delay=%d abort=%d",
+              elem->channel_data, calld, calld->delay_request_,
+              calld->abort_request_);
+    }
+    if (calld->MaybeDelay()) {
+      // Delay the batch, and pass down the batch in the scheduled closure.
+      calld->DelayBatch(elem, batch);
+      return;
+    }
+    grpc_error* abort_error = calld->MaybeAbort();
+    if (abort_error != GRPC_ERROR_NONE) {
+      calld->abort_error_ = abort_error;
+      grpc_transport_stream_op_batch_finish_with_failure(
+          batch, GRPC_ERROR_REF(calld->abort_error_), calld->call_combiner_);
+      return;
+    }
+  } else {
+    if (batch->recv_trailing_metadata) {
+      // Intercept recv_trailing_metadata callback so that we can inject the
+      // failure when aborting streaming calls, because their
+      // recv_trailing_metatdata op may not be on the same batch as the
+      // send_initial_metadata op.
+      calld->original_recv_trailing_metadata_ready_ =
+          batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready;
+      batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready =
+          &calld->recv_trailing_metadata_ready_;
+    }
+    if (calld->abort_error_ != GRPC_ERROR_NONE) {
+      // If we already decided to abort, then immediately fail this batch.
+      grpc_transport_stream_op_batch_finish_with_failure(
+          batch, GRPC_ERROR_REF(calld->abort_error_), calld->call_combiner_);
+      return;
+    }
+  }
+  // Chain to the next filter.
+  grpc_call_next_op(elem, batch);
+}
+
+CallData::CallData(grpc_call_element* elem, const grpc_call_element_args* args)
+    : owning_call_(args->call_stack),
+      arena_(args->arena),
+      call_combiner_(args->call_combiner) {
+  auto* chand = static_cast<ChannelData*>(elem->channel_data);
+  // Fetch the fault injection policy from the service config, based on the
+  // relative index for which policy should this CallData use.
+  auto* service_config_call_data = static_cast<ServiceConfigCallData*>(
+      args->context[GRPC_CONTEXT_SERVICE_CONFIG_CALL_DATA].value);
+  auto* method_params = static_cast<FaultInjectionMethodParsedConfig*>(
+      service_config_call_data->GetMethodParsedConfig(
+          FaultInjectionServiceConfigParser::ParserIndex()));
+  if (method_params != nullptr) {
+    fi_policy_ = method_params->fault_injection_policy(chand->index());
+  }
+  GRPC_CLOSURE_INIT(&recv_trailing_metadata_ready_,
+                    HijackedRecvTrailingMetadataReady, elem,
+                    grpc_schedule_on_exec_ctx);
+}
+
+CallData::~CallData() {
+  if (fi_policy_owned_) {
+    fi_policy_->~FaultInjectionPolicy();
+  }
+  GRPC_ERROR_UNREF(abort_error_);
+}
+
+void CallData::DecideWhetherToInjectFaults(
+    grpc_metadata_batch* initial_metadata) {
+  FaultInjectionMethodParsedConfig::FaultInjectionPolicy* copied_policy =
+      nullptr;
+  // Update the policy with values in initial metadata.
+  if (!fi_policy_->abort_code_header.empty() ||
+      !fi_policy_->abort_percentage_header.empty() ||
+      !fi_policy_->delay_header.empty() ||
+      !fi_policy_->delay_percentage_header.empty()) {
+    // Defer the actual copy until the first matched header.
+    auto maybe_copy_policy_func = [this, &copied_policy]() {
+      if (copied_policy == nullptr) {
+        copied_policy =
+            arena_->New<FaultInjectionMethodParsedConfig::FaultInjectionPolicy>(
+                *fi_policy_);
+      }
+    };
+    for (grpc_linked_mdelem* md = initial_metadata->list.head; md != nullptr;
+         md = md->next) {
+      absl::string_view key = StringViewFromSlice(GRPC_MDKEY(md->md));
+      // Only perform string comparison if:
+      //   1. Needs to check this header;
+      //   2. The value is not been filled before.
+      if (!fi_policy_->abort_code_header.empty() &&
+          (copied_policy == nullptr ||
+           copied_policy->abort_code == GRPC_STATUS_OK) &&
+          key == fi_policy_->abort_code_header) {
+        maybe_copy_policy_func();
+        grpc_status_code_from_int(GetLinkedMetadatumValueInt(md),
+                                  &copied_policy->abort_code);
+      }
+      if (!fi_policy_->abort_percentage_header.empty() &&
+          key == fi_policy_->abort_percentage_header) {
+        maybe_copy_policy_func();
+        copied_policy->abort_percentage_numerator =
+            GPR_MIN(GetLinkedMetadatumValueUnsignedInt(md),
+                    fi_policy_->abort_percentage_numerator);
+      }
+      if (!fi_policy_->delay_header.empty() &&
+          (copied_policy == nullptr || copied_policy->delay == 0) &&
+          key == fi_policy_->delay_header) {
+        maybe_copy_policy_func();
+        copied_policy->delay = static_cast<grpc_millis>(
+            GPR_MAX(GetLinkedMetadatumValueInt64(md), 0));
+      }
+      if (!fi_policy_->delay_percentage_header.empty() &&
+          key == fi_policy_->delay_percentage_header) {
+        maybe_copy_policy_func();
+        copied_policy->delay_percentage_numerator =
+            GPR_MIN(GetLinkedMetadatumValueUnsignedInt(md),
+                    fi_policy_->delay_percentage_numerator);
+      }
+    }
+    if (copied_policy != nullptr) fi_policy_ = copied_policy;
+  }
+  // Roll the dice
+  delay_request_ = fi_policy_->delay != 0 &&
+                   UnderFraction(fi_policy_->delay_percentage_numerator,
+                                 fi_policy_->delay_percentage_denominator);
+  abort_request_ = fi_policy_->abort_code != GRPC_STATUS_OK &&
+                   UnderFraction(fi_policy_->abort_percentage_numerator,
+                                 fi_policy_->abort_percentage_denominator);
+  if (!delay_request_ && !abort_request_) {
+    if (copied_policy != nullptr) copied_policy->~FaultInjectionPolicy();
+    // No fault injection for this call
+  } else {
+    fi_policy_owned_ = copied_policy != nullptr;
+  }
+}
+
+bool CallData::HaveActiveFaultsQuota(bool increment) {
+  if (g_active_faults.Load(MemoryOrder::ACQUIRE) >= fi_policy_->max_faults) {
+    return false;
+  }
+  if (increment) g_active_faults.FetchAdd(1, MemoryOrder::RELAXED);
+  return true;
+}
+
+bool CallData::MaybeDelay() {
+  if (delay_request_) {
+    return HaveActiveFaultsQuota(true);
+  }
+  return false;
+}
+
+grpc_error* CallData::MaybeAbort() {
+  if (abort_request_ && (delay_request_ || HaveActiveFaultsQuota(false))) {
+    return grpc_error_set_int(
+        GRPC_ERROR_CREATE_FROM_COPIED_STRING(fi_policy_->abort_message.c_str()),
+        GRPC_ERROR_INT_GRPC_STATUS, fi_policy_->abort_code);
+  }
+  return GRPC_ERROR_NONE;
+}
+
+void CallData::DelayBatch(grpc_call_element* elem,
+                          grpc_transport_stream_op_batch* batch) {
+  MutexLock lock(&delay_mu_);
+  delayed_batch_ = batch;
+  resume_batch_canceller_ = new ResumeBatchCanceller(elem);
+  grpc_millis resume_time = ExecCtx::Get()->Now() + fi_policy_->delay;
+  GRPC_CLOSURE_INIT(&batch->handler_private.closure, ResumeBatch, elem,
+                    grpc_schedule_on_exec_ctx);
+  grpc_timer_init(&delay_timer_, resume_time, &batch->handler_private.closure);
+}
+
+void CallData::ResumeBatch(void* arg, grpc_error* error) {
+  grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
+  auto* calld = static_cast<CallData*>(elem->call_data);
+  MutexLock lock(&calld->delay_mu_);
+  // Cancelled or canceller has already run
+  if (error == GRPC_ERROR_CANCELLED ||
+      calld->resume_batch_canceller_ == nullptr) {
+    return;
+  }
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_fault_injection_filter_trace)) {
+    gpr_log(GPR_INFO, "chand=%p calld=%p: Resuming delayed stream op batch %p",
+            elem->channel_data, calld, calld->delayed_batch_);
+  }
+  // Lame the canceller
+  calld->resume_batch_canceller_ = nullptr;
+  // Finish fault injection.
+  calld->FaultInjectionFinished();
+  // Abort if needed.
+  error = calld->MaybeAbort();
+  if (error != GRPC_ERROR_NONE) {
+    grpc_transport_stream_op_batch_finish_with_failure(
+        calld->delayed_batch_, error, calld->call_combiner_);
+    return;
+  }
+  // Chain to the next filter.
+  grpc_call_next_op(elem, calld->delayed_batch_);
+}
+
+void CallData::HijackedRecvTrailingMetadataReady(void* arg, grpc_error* error) {
+  grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
+  auto* calld = static_cast<CallData*>(elem->call_data);
+  if (calld->abort_error_ != GRPC_ERROR_NONE) {
+    error = grpc_error_add_child(GRPC_ERROR_REF(error),
+                                 GRPC_ERROR_REF(calld->abort_error_));
+    Closure::Run(DEBUG_LOCATION, calld->original_recv_trailing_metadata_ready_,
+                 error);
+  }
+}
+
+}  // namespace
+
+extern const grpc_channel_filter FaultInjectionFilterVtable = {
+    CallData::StartTransportStreamOpBatch,
+    grpc_channel_next_op,
+    sizeof(CallData),
+    CallData::Init,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    CallData::Destroy,
+    sizeof(ChannelData),
+    ChannelData::Init,
+    ChannelData::Destroy,
+    grpc_channel_next_get_info,
+    "fault_injection_filter",
+};
+
+void FaultInjectionFilterInit(void) {
+  grpc_core::FaultInjectionServiceConfigParser::Register();
+}
+
+void FaultInjectionFilterShutdown(void) {}
+
+}  // namespace grpc_core