grpc 1.66.0 → 1.67.0.pre1

data/src/core/util/latent_see.cc

@@ -15,24 +15,45 @@
 #include "src/core/util/latent_see.h"
 
 #ifdef GRPC_ENABLE_LATENT_SEE
+#include <atomic>
 #include <chrono>
 #include <cstdint>
+#include <string>
+#include <vector>
 
 #include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 
+#include "src/core/lib/gprpp/sync.h"
+#include "src/core/util/ring_buffer.h"
+
 namespace grpc_core {
 namespace latent_see {
 
-thread_local std::vector<Log::Event> Log::thread_events_;
 thread_local uint64_t Log::thread_id_ = Log::Get().next_thread_id_.fetch_add(1);
+thread_local Bin* Log::bin_ = nullptr;
+thread_local void* Log::bin_owner_ = nullptr;
 std::atomic<uint64_t> Flow::next_flow_id_{1};
+std::atomic<uintptr_t> Log::free_bins_{0};
 
 std::string Log::GenerateJson() {
   std::vector<RecordedEvent> events;
+  RingBuffer<RecordedEvent, Log::kMaxEventsPerCpu>* other;
   for (auto& fragment : fragments_) {
-    MutexLock lock(&fragment.mu);
-    events.insert(events.end(), fragment.events.begin(), fragment.events.end());
+    {
+      MutexLock lock(&fragment.mu);
+      other = fragment.active;
+      if (fragment.active == &fragment.primary) {
+        fragment.active = &fragment.secondary;
+      } else {
+        fragment.active = &fragment.primary;
+      }
+    }
+    for (auto it = other->begin(); it != other->end(); ++it) {
+      events.push_back(*it);
+    }
+    other->Clear();
   }
   absl::optional<std::chrono::steady_clock::time_point> start_time;
   for (auto& event : events) {
@@ -91,9 +112,8 @@ std::string Log::GenerateJson() {
   return json;
 }
 
-void Log::FlushThreadLog() {
-  auto& thread_events = thread_events_;
-  if (thread_events.empty()) return;
+void Log::FlushBin(Bin* bin) {
+  if (bin->events.empty()) return;
   auto& log = Get();
   const auto batch_id =
       log.next_batch_id_.fetch_add(1, std::memory_order_relaxed);
@@ -101,11 +121,11 @@ void Log::FlushThreadLog() {
   const auto thread_id = thread_id_;
   {
     MutexLock lock(&fragment.mu);
-    for (auto event : thread_events) {
-      fragment.events.push_back(RecordedEvent{thread_id, batch_id, event});
+    for (auto event : bin->events) {
+      fragment.active->Append(RecordedEvent{thread_id, batch_id, event});
     }
   }
-  thread_events.clear();
+  bin->events.clear();
 }
 
 }  // namespace latent_see

data/src/core/util/latent_see.h

@@ -18,15 +18,28 @@
 #include <grpc/support/port_platform.h>
 
 #ifdef GRPC_ENABLE_LATENT_SEE
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#include <atomic>
 #include <chrono>
 #include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <string>
 #include <utility>
 #include <vector>
 
+#include "absl/base/thread_annotations.h"
+#include "absl/functional/any_invocable.h"
 #include "absl/log/log.h"
+#include "absl/strings/string_view.h"
 
 #include "src/core/lib/gprpp/per_cpu.h"
 #include "src/core/lib/gprpp/sync.h"
+#include "src/core/util/ring_buffer.h"
+
+#define TAGGED_POINTER_SIZE_BITS 48
 
 namespace grpc_core {
 namespace latent_see {
@@ -39,22 +52,74 @@ struct Metadata {
 
 enum class EventType : uint8_t { kBegin, kEnd, kFlowStart, kFlowEnd, kMark };
 
+// A bin collects all events that occur within a parent scope.
+struct Bin {
+  struct Event {
+    const Metadata* metadata;
+    std::chrono::steady_clock::time_point timestamp;
+    uint64_t id;
+    EventType type;
+  };
+
+  void Append(const Metadata* metadata, EventType type, uint64_t id) {
+    events.push_back(
+        Event{metadata, std::chrono::steady_clock::now(), id, type});
+  }
+
+  std::vector<Event> events;
+  uintptr_t next_free = 0;
+};
+
 class Log {
  public:
-  static void FlushThreadLog();
+  static constexpr int kMaxEventsPerCpu = 50000;
+  static constexpr uintptr_t kTagMask = (1ULL << TAGGED_POINTER_SIZE_BITS) - 1;
 
-  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static void Append(
-      const Metadata* metadata, EventType type, uint64_t id) {
-    thread_events_.push_back(
-        Event{metadata, std::chrono::steady_clock::now(), id, type});
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static uintptr_t IncrementTag(
+      uintptr_t input) {
+    return input + (1UL << TAGGED_POINTER_SIZE_BITS);
   }
 
- private:
-  Log() = default;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Bin* ToBin(uintptr_t ptr) {
+    return reinterpret_cast<Bin*>(ptr & kTagMask);
+  }
+
+  static uintptr_t StartBin(void* owner) {
+    uintptr_t bin_descriptor = free_bins_.load(std::memory_order_acquire);
+    Bin* bin;
+    do {
+      if (bin_descriptor == 0) {
+        bin = new Bin();
+        break;
+      }
+      bin = ToBin(bin_descriptor);
+    } while (!free_bins_.compare_exchange_strong(bin_descriptor, bin->next_free,
+                                                 std::memory_order_acq_rel));
+    bin_ = bin;
+    bin_owner_ = owner;
+    return reinterpret_cast<uintptr_t>(bin);
+  }
+
+  static void EndBin(uintptr_t bin_descriptor, void* owner) {
+    if (bin_owner_ != owner || bin_descriptor == 0) return;
+    FlushBin(ToBin(bin_descriptor));
+    uintptr_t next_free = free_bins_.load(std::memory_order_acquire);
+    while (!free_bins_.compare_exchange_strong(
+        next_free, IncrementTag(bin_descriptor), std::memory_order_acq_rel)) {
+    }
+    bin_ = nullptr;
+    bin_owner_ = nullptr;
+  }
+
+  static Bin* CurrentThreadBin() { return bin_; }
 
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Log& Get() {
     static Log* log = []() {
       atexit([] {
+        if (log->stats_flusher_ != nullptr) {
+          log->stats_flusher_(log->GenerateJson());
+          return;
+        }
         LOG(INFO) << "Writing latent_see.json in " << get_current_dir_name();
         FILE* f = fopen("latent_see.json", "w");
         if (f == nullptr) return;
@@ -68,38 +133,55 @@ class Log {
 
   std::string GenerateJson();
 
-  struct Event {
-    const Metadata* metadata;
-    std::chrono::steady_clock::time_point timestamp;
-    uint64_t id;
-    EventType type;
-  };
+  void OverrideStatsFlusher(
+      absl::AnyInvocable<void(absl::string_view)> stats_exporter) {
+    stats_flusher_ = std::move(stats_exporter);
+  }
+
+ private:
+  Log() = default;
+
+  static void FlushBin(Bin* bin);
+
   struct RecordedEvent {
     uint64_t thread_id;
     uint64_t batch_id;
-    Event event;
+    Bin::Event event;
   };
   std::atomic<uint64_t> next_thread_id_{1};
   std::atomic<uint64_t> next_batch_id_{1};
-  static thread_local std::vector<Event> thread_events_;
   static thread_local uint64_t thread_id_;
+  static thread_local Bin* bin_;
+  static thread_local void* bin_owner_;
+  static std::atomic<uintptr_t> free_bins_;
+  absl::AnyInvocable<void(absl::string_view)> stats_flusher_ = nullptr;
   struct Fragment {
+    Fragment() : active(&primary) {};
     Mutex mu;
-    std::vector<RecordedEvent> events ABSL_GUARDED_BY(mu);
+    RingBuffer<RecordedEvent, Log::kMaxEventsPerCpu>* active
+        ABSL_GUARDED_BY(mu);
+    RingBuffer<RecordedEvent, Log::kMaxEventsPerCpu> primary;
+    RingBuffer<RecordedEvent, Log::kMaxEventsPerCpu> secondary;
   };
   PerCpu<Fragment> fragments_{PerCpuOptions()};
 };
 
-template <bool kFlush>
+template <bool kParent>
 class Scope {
  public:
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Scope(const Metadata* metadata)
       : metadata_(metadata) {
-    Log::Append(metadata_, EventType::kBegin, 0);
+    bin_ = Log::CurrentThreadBin();
+    if (kParent && bin_ == nullptr) {
+      bin_descriptor_ = Log::StartBin(this);
+      bin_ = Log::ToBin(bin_descriptor_);
+    }
+    CHECK_NE(bin_, nullptr);
+    bin_->Append(metadata_, EventType::kBegin, 0);
   }
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~Scope() {
-    Log::Append(metadata_, EventType::kEnd, 0);
-    if (kFlush) Log::FlushThreadLog();
+    bin_->Append(metadata_, EventType::kEnd, 0);
+    if (kParent) Log::EndBin(bin_descriptor_, this);
   }
 
   Scope(const Scope&) = delete;
@@ -107,6 +189,8 @@ class Scope {
 
  private:
   const Metadata* const metadata_;
+  uintptr_t bin_descriptor_ = 0;
+  Bin* bin_ = nullptr;
 };
 
 using ParentScope = Scope<true>;
@@ -118,11 +202,11 @@ class Flow {
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION explicit Flow(const Metadata* metadata)
       : metadata_(metadata),
         id_(next_flow_id_.fetch_add(1, std::memory_order_relaxed)) {
-    Log::Append(metadata_, EventType::kFlowStart, id_);
+    Log::CurrentThreadBin()->Append(metadata_, EventType::kFlowStart, id_);
   }
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION ~Flow() {
     if (metadata_ != nullptr) {
-      Log::Append(metadata_, EventType::kFlowEnd, id_);
+      Log::CurrentThreadBin()->Append(metadata_, EventType::kFlowEnd, id_);
     }
   }
 
@@ -131,7 +215,9 @@ class Flow {
   Flow(Flow&& other) noexcept
       : metadata_(std::exchange(other.metadata_, nullptr)), id_(other.id_) {}
   Flow& operator=(Flow&& other) noexcept {
-    if (metadata_ != nullptr) Log::Append(metadata_, EventType::kFlowEnd, id_);
+    if (metadata_ != nullptr) {
+      Log::CurrentThreadBin()->Append(metadata_, EventType::kFlowEnd, id_);
+    }
     metadata_ = std::exchange(other.metadata_, nullptr);
     id_ = other.id_;
     return *this;
@@ -142,15 +228,18 @@ class Flow {
   }
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void End() {
     if (metadata_ == nullptr) return;
-    Log::Append(metadata_, EventType::kFlowEnd, id_);
+    Log::CurrentThreadBin()->Append(metadata_, EventType::kFlowEnd, id_);
     metadata_ = nullptr;
   }
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void Begin(const Metadata* metadata) {
-    if (metadata_ != nullptr) Log::Append(metadata_, EventType::kFlowEnd, id_);
+    auto* bin = Log::CurrentThreadBin();
+    if (metadata_ != nullptr) {
+      bin->Append(metadata_, EventType::kFlowEnd, id_);
+    }
     metadata_ = metadata;
     if (metadata_ == nullptr) return;
     id_ = next_flow_id_.fetch_add(1, std::memory_order_relaxed);
-    Log::Append(metadata_, EventType::kFlowStart, id_);
+    bin->Append(metadata_, EventType::kFlowStart, id_);
   }
 
  private:
@@ -160,7 +249,7 @@ class Flow {
 };
 
 GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline void Mark(const Metadata* md) {
-  Log::Append(md, EventType::kMark, 0);
+  Log::CurrentThreadBin()->Append(md, EventType::kMark, 0);
 }
 
 }  // namespace latent_see
@@ -197,6 +286,12 @@ struct Flow {
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void End() {}
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void Begin(Metadata*) {}
 };
+struct ParentScope {
+  explicit ParentScope(Metadata*) {}
+};
+struct InnerScope {
+  explicit InnerScope(Metadata*) {}
+};
 }  // namespace latent_see
 }  // namespace grpc_core
 #define GRPC_LATENT_SEE_METADATA(name) nullptr

data/src/core/util/log.cc

@@ -23,6 +23,7 @@
 #include <stdio.h>
 #include <string.h>
 
+#include "absl/log/check.h"
 #include "absl/log/globals.h"
 #include "absl/strings/match.h"
 #include "absl/strings/str_cat.h"
@@ -35,79 +36,66 @@
 #include "src/core/lib/gprpp/crash.h"
 #include "src/core/util/string.h"
 
-void gpr_default_log(gpr_log_func_args* args);
-void gpr_platform_log(gpr_log_func_args* args);
-static gpr_atm g_log_func = reinterpret_cast<gpr_atm>(gpr_default_log);
-
 void gpr_unreachable_code(const char* reason, const char* file, int line) {
   grpc_core::Crash(absl::StrCat("UNREACHABLE CODE: ", reason),
                    grpc_core::SourceLocation(file, line));
 }
 
-int gpr_should_log(gpr_log_severity severity) {
+GPRAPI void grpc_absl_log(const char* file, int line, gpr_log_severity severity,
+                          const char* message_str) {
   switch (severity) {
-    case GPR_LOG_SEVERITY_ERROR:
-      return absl::MinLogLevel() <= absl::LogSeverityAtLeast::kError;
-    case GPR_LOG_SEVERITY_INFO:
-      // There is no documentation about how expensive or inexpensive
-      // MinLogLevel is. We could have saved this in a static const variable.
-      // But decided against it just in case anyone programatically sets absl
-      // min log level settings after this has been initialized.
-      // Same holds for ABSL_VLOG_IS_ON(2).
-      return absl::MinLogLevel() <= absl::LogSeverityAtLeast::kInfo;
     case GPR_LOG_SEVERITY_DEBUG:
-      return ABSL_VLOG_IS_ON(2);
+      VLOG(2).AtLocation(file, line) << message_str;
+      return;
+    case GPR_LOG_SEVERITY_INFO:
+      LOG(INFO).AtLocation(file, line) << message_str;
+      return;
+    case GPR_LOG_SEVERITY_ERROR:
+      LOG(ERROR).AtLocation(file, line) << message_str;
+      return;
     default:
-      DLOG(ERROR) << "Invalid gpr_log_severity.";
-      return true;
+      DCHECK(false) << "Invalid severity";
   }
 }
 
-void gpr_default_log(gpr_log_func_args* args) {
-  switch (args->severity) {
+GPRAPI void grpc_absl_log_int(const char* file, int line,
+                              gpr_log_severity severity,
+                              const char* message_str, intptr_t num) {
+  switch (severity) {
     case GPR_LOG_SEVERITY_DEBUG:
-      //  Log DEBUG messages as VLOG(2).
-      VLOG(2).AtLocation(args->file, args->line) << args->message;
+      VLOG(2).AtLocation(file, line) << message_str << num;
       return;
     case GPR_LOG_SEVERITY_INFO:
-      LOG(INFO).AtLocation(args->file, args->line) << args->message;
+      LOG(INFO).AtLocation(file, line) << message_str << num;
       return;
     case GPR_LOG_SEVERITY_ERROR:
-      LOG(ERROR).AtLocation(args->file, args->line) << args->message;
+      LOG(ERROR).AtLocation(file, line) << message_str << num;
       return;
     default:
-      LOG(ERROR) << __func__ << ": unknown gpr log severity(" << args->severity
-                 << "), using ERROR";
-      LOG(ERROR).AtLocation(args->file, args->line) << args->message;
+      DCHECK(false) << "Invalid severity";
   }
 }
 
-void gpr_log_message(const char* file, int line, gpr_log_severity severity,
-                     const char* message) {
-  if (gpr_should_log(severity) == 0) {
-    return;
+GPRAPI void grpc_absl_log_str(const char* file, int line,
+                              gpr_log_severity severity,
+                              const char* message_str1,
+                              const char* message_str2) {
+  switch (severity) {
+    case GPR_LOG_SEVERITY_DEBUG:
+      VLOG(2).AtLocation(file, line) << message_str1 << message_str2;
+      return;
+    case GPR_LOG_SEVERITY_INFO:
+      LOG(INFO).AtLocation(file, line) << message_str1 << message_str2;
+      return;
+    case GPR_LOG_SEVERITY_ERROR:
+      LOG(ERROR).AtLocation(file, line) << message_str1 << message_str2;
+      return;
+    default:
+      DCHECK(false) << "Invalid severity";
   }
-
-  gpr_log_func_args lfargs;
-  memset(&lfargs, 0, sizeof(lfargs));
-  lfargs.file = file;
-  lfargs.line = line;
-  lfargs.severity = severity;
-  lfargs.message = message;
-  reinterpret_cast<gpr_log_func>(gpr_atm_no_barrier_load(&g_log_func))(&lfargs);
-}
-
-void gpr_set_log_verbosity(
-    [[maybe_unused]] gpr_log_severity deprecated_setting) {
-  LOG(ERROR)
-      << "This will not be set. Please set this via absl log level settings.";
 }
 
 void gpr_log_verbosity_init(void) {
-// This is enabled in Github only.
-// This ifndef is converted to ifdef internally by copybara.
-// Internally grpc verbosity is managed using absl settings.
-// So internally we avoid setting it like this.
 #ifndef GRPC_VERBOSITY_MACRO
   // SetMinLogLevel sets the value for the entire binary, not just gRPC.
   // This setting will change things for other libraries/code that is unrelated
@@ -140,10 +128,3 @@ void gpr_log_verbosity_init(void) {
   }
 #endif  // GRPC_VERBOSITY_MACRO
 }
-
-void gpr_set_log_function([[maybe_unused]] gpr_log_func deprecated_setting) {
-  LOG(ERROR)
-      << "This function is deprecated. This function will be deleted in the "
-         "next gRPC release. You may create a new absl LogSink with similar "
-         "functionality. gRFC: https://github.com/grpc/proposal/pull/425 ";
-}

data/src/core/util/lru_cache.h

@@ -0,0 +1,104 @@
+//
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+#ifndef GRPC_SRC_CORE_UTIL_LRU_CACHE_H
+#define GRPC_SRC_CORE_UTIL_LRU_CACHE_H
+
+#include <list>
+#include <tuple>
+#include <utility>
+
+#include "absl/container/flat_hash_map.h"
+#include "absl/functional/any_invocable.h"
+#include "absl/log/check.h"
+#include "absl/types/optional.h"
+
+namespace grpc_core {
+
+// A simple LRU cache.  Retains at most max_size entries.
+// Caller is responsible for synchronization.
+// TODO(roth): Support heterogenous lookups.
+template <typename Key, typename Value>
+class LruCache {
+ public:
+  explicit LruCache(size_t max_size) : max_size_(max_size) {
+    CHECK_GT(max_size, 0UL);
+  }
+
+  // Returns the value for key, or nullopt if not present.
+  absl::optional<Value> Get(Key key);
+
+  // If key is present in the cache, returns the corresponding value.
+  // Otherwise, inserts a new entry in the map, calling create() to
+  // construct the new value.  If inserting a new entry causes the cache
+  // to be too large, removes the least recently used entry.
+  Value GetOrInsert(Key key, absl::AnyInvocable<Value(const Key&)> create);
+
+ private:
+  struct CacheEntry {
+    Value value;
+    typename std::list<Key>::iterator lru_iterator;
+
+    explicit CacheEntry(Value v) : value(std::move(v)) {}
+  };
+
+  const size_t max_size_;
+  absl::flat_hash_map<Key, CacheEntry> cache_;
+  std::list<Key> lru_list_;
+};
+
+//
+// implementation -- no user-serviceable parts below
+//
+
+template <typename Key, typename Value>
+absl::optional<Value> LruCache<Key, Value>::Get(Key key) {
+  auto it = cache_.find(key);
+  if (it == cache_.end()) return absl::nullopt;
+  // Found the entry.  Move the entry to the end of the LRU list.
+  auto new_lru_it = lru_list_.insert(lru_list_.end(), *it->second.lru_iterator);
+  lru_list_.erase(it->second.lru_iterator);
+  it->second.lru_iterator = new_lru_it;
+  return it->second.value;
+}
+
+template <typename Key, typename Value>
+Value LruCache<Key, Value>::GetOrInsert(
+    Key key, absl::AnyInvocable<Value(const Key&)> create) {
+  auto value = Get(key);
+  if (value.has_value()) return std::move(*value);
+  // Entry not found.  We'll need to insert a new entry.
+  // If the cache is at max size, remove the least recently used entry.
+  if (cache_.size() == max_size_) {
+    auto lru_it = lru_list_.begin();
+    CHECK(lru_it != lru_list_.end());
+    auto cache_it = cache_.find(*lru_it);
+    CHECK(cache_it != cache_.end());
+    cache_.erase(cache_it);
+    lru_list_.pop_front();
+  }
+  // Create a new entry, insert it, and return it.
+  auto it = cache_
+                .emplace(std::piecewise_construct, std::forward_as_tuple(key),
+                         std::forward_as_tuple(create(key)))
+                .first;
+  it->second.lru_iterator = lru_list_.insert(lru_list_.end(), std::move(key));
+  return it->second.value;
+}
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_UTIL_LRU_CACHE_H

data/src/core/util/msys/tmpfile.cc

@@ -26,7 +26,6 @@
 #include <tchar.h>
 
 #include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/gprpp/crash.h"

data/src/core/util/posix/sync.cc

@@ -27,7 +27,6 @@
 #include "absl/log/check.h"
 
 #include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
 

data/src/core/util/posix/time.cc

@@ -31,7 +31,6 @@
 #include "absl/log/check.h"
 
 #include <grpc/support/atm.h>
-#include <grpc/support/log.h>
 #include <grpc/support/time.h>
 
 #include "src/core/lib/gprpp/crash.h"