grpc 1.40.0 → 1.43.1

data/src/core/ext/transport/chttp2/transport/hpack_encoder.h

@@ -23,85 +23,287 @@
 
 #include <grpc/slice.h>
 #include <grpc/slice_buffer.h>
+
 #include "src/core/ext/transport/chttp2/transport/frame.h"
+#include "src/core/ext/transport/chttp2/transport/hpack_encoder_index.h"
+#include "src/core/ext/transport/chttp2/transport/hpack_encoder_table.h"
+#include "src/core/ext/transport/chttp2/transport/popularity_count.h"
 #include "src/core/lib/transport/metadata.h"
 #include "src/core/lib/transport/metadata_batch.h"
 #include "src/core/lib/transport/transport.h"
 
-// This should be <= 8. We use 6 to save space.
-#define GRPC_CHTTP2_HPACKC_NUM_VALUES_BITS 6
-#define GRPC_CHTTP2_HPACKC_NUM_VALUES (1 << GRPC_CHTTP2_HPACKC_NUM_VALUES_BITS)
-/* initial table size, per spec */
-#define GRPC_CHTTP2_HPACKC_INITIAL_TABLE_SIZE 4096
-/* maximum table size we'll actually use */
-#define GRPC_CHTTP2_HPACKC_MAX_TABLE_SIZE (1024 * 1024)
-
 extern grpc_core::TraceFlag grpc_http_trace;
 
-struct grpc_chttp2_hpack_compressor {
-  uint32_t max_table_size;
-  uint32_t max_table_elems;
-  uint32_t cap_table_elems;
-  /** maximum number of bytes we'll use for the decode table (to guard against
-      peers ooming us by setting decode table size high) */
-  uint32_t max_usable_size;
-  /* one before the lowest usable table index */
-  uint32_t tail_remote_index;
-  uint32_t table_size;
-  uint32_t table_elems;
-  uint16_t* table_elem_size;
-  /** if non-zero, advertise to the decoder that we'll start using a table
-      of this size */
-  uint8_t advertise_table_size_change;
-
-  /* filter tables for elems: this tables provides an approximate
-     popularity count for particular hashes, and are used to determine whether
-     a new literal should be added to the compression table or not.
-     They track a single integer that counts how often a particular value has
-     been seen. When that count reaches max (255), all values are halved. */
-  uint32_t filter_elems_sum;
-  uint8_t filter_elems[GRPC_CHTTP2_HPACKC_NUM_VALUES];
-
-  /* entry tables for keys & elems: these tables track values that have been
-     seen and *may* be in the decompressor table */
-  struct {
-    struct {
-      grpc_mdelem value;
-      uint32_t index;
-    } entries[GRPC_CHTTP2_HPACKC_NUM_VALUES];
-  } elem_table; /* Metadata table management */
-  struct {
-    struct {
-      /* Only store the slice refcount - we do not need the byte buffer or
-         length of the slice since we only need to store a mapping between the
-         identity of the slice and the corresponding HPACK index. Since the
-         slice *must* be static or interned, the refcount is sufficient to
-         establish identity. */
-      grpc_slice_refcount* value;
-      uint32_t index;
-    } entries[GRPC_CHTTP2_HPACKC_NUM_VALUES];
-  } key_table; /* Key table management */
+namespace grpc_core {
+
+// Wrapper to take an array of mdelems and make them encodable
+class MetadataArray {
+ public:
+  MetadataArray(grpc_mdelem** elems, size_t count)
+      : elems_(elems), count_(count) {}
+
+  template <typename Encoder>
+  void Encode(Encoder* encoder) const {
+    for (size_t i = 0; i < count_; i++) {
+      encoder->Encode(*elems_[i]);
+    }
+  }
+
+ private:
+  grpc_mdelem** elems_;
+  size_t count_;
+};
+
+namespace metadata_detail {
+template <typename A, typename B>
+class ConcatMetadata {
+ public:
+  ConcatMetadata(const A& a, const B& b) : a_(a), b_(b) {}
+
+  template <typename Encoder>
+  void Encode(Encoder* encoder) const {
+    a_.Encode(encoder);
+    b_.Encode(encoder);
+  }
+
+ private:
+  const A& a_;
+  const B& b_;
 };
+}  // namespace metadata_detail
+
+template <typename A, typename B>
+metadata_detail::ConcatMetadata<A, B> ConcatMetadata(const A& a, const B& b) {
+  return metadata_detail::ConcatMetadata<A, B>(a, b);
+}
+
+class HPackCompressor {
+ public:
+  HPackCompressor() = default;
+  ~HPackCompressor() = default;
+
+  // Maximum table size we'll actually use.
+  static constexpr uint32_t kMaxTableSize = 1024 * 1024;
+
+  void SetMaxTableSize(uint32_t max_table_size);
+  void SetMaxUsableSize(uint32_t max_table_size);
+
+  uint32_t test_only_table_size() const {
+    return table_.test_only_table_size();
+  }
+
+  struct EncodeHeaderOptions {
+    uint32_t stream_id;
+    bool is_end_of_stream;
+    bool use_true_binary_metadata;
+    size_t max_frame_size;
+    grpc_transport_one_way_stats* stats;
+  };
+
+  template <typename HeaderSet>
+  void EncodeHeaders(const EncodeHeaderOptions& options,
+                     const HeaderSet& headers, grpc_slice_buffer* output) {
+    Framer framer(options, this, output);
+    headers.Encode(&framer);
+  }
+
+  class Framer {
+   public:
+    Framer(const EncodeHeaderOptions& options, HPackCompressor* compressor,
+           grpc_slice_buffer* output);
+    ~Framer() { FinishFrame(true); }
+
+    Framer(const Framer&) = delete;
+    Framer& operator=(const Framer&) = delete;
+
+    void Encode(grpc_mdelem md);
+    void Encode(GrpcTimeoutMetadata, grpc_millis deadline);
+    void Encode(TeMetadata, TeMetadata::ValueType value);
+    void Encode(UserAgentMetadata, const Slice& slice);
+    void Encode(GrpcMessageMetadata, const Slice& slice) {
+      if (slice.empty()) return;
+      EmitLitHdrWithNonBinaryStringKeyNotIdx(
+          StaticSlice::FromStaticString("grpc-message").c_slice(),
+          slice.c_slice());
+    }
+    template <typename Which>
+    void Encode(Which, const Slice& slice) {
+      if (absl::EndsWith(Which::key(), "-bin")) {
+        EmitLitHdrWithBinaryStringKeyNotIdx(
+            StaticSlice::FromStaticString(Which::key()).c_slice(),
+            slice.c_slice());
+      } else {
+        EmitLitHdrWithNonBinaryStringKeyNotIdx(
+            StaticSlice::FromStaticString(Which::key()).c_slice(),
+            slice.c_slice());
+      }
+    }
+
+   private:
+    struct FramePrefix {
+      // index (in output_) of the header for the frame
+      size_t header_idx;
+      // number of bytes in 'output' when we started the frame - used to
+      // calculate frame length
+      size_t output_length_at_start_of_frame;
+    };
+
+    FramePrefix BeginFrame();
+    void FinishFrame(bool is_header_boundary);
+    void EnsureSpace(size_t need_bytes);
+
+    void AdvertiseTableSizeChange();
+    void EmitIndexed(uint32_t index);
+    void EncodeDynamic(grpc_mdelem elem);
+    static GPR_ATTRIBUTE_NOINLINE void Log(grpc_mdelem elem);
+
+    void EmitLitHdrIncIdx(uint32_t key_index, grpc_mdelem elem);
+    void EmitLitHdrNotIdx(uint32_t key_index, grpc_mdelem elem);
+    void EmitLitHdrWithStringKeyIncIdx(grpc_mdelem elem);
+    void EmitLitHdrWithNonBinaryStringKeyIncIdx(const grpc_slice& key_slice,
+                                                const grpc_slice& value_slice);
+    void EmitLitHdrWithBinaryStringKeyNotIdx(const grpc_slice& key_slice,
+                                             const grpc_slice& value_slice);
+    void EmitLitHdrWithNonBinaryStringKeyNotIdx(const grpc_slice& key_slice,
+                                                const grpc_slice& value_slice);
+    void EmitLitHdrWithStringKeyNotIdx(grpc_mdelem elem);
 
-void grpc_chttp2_hpack_compressor_init(grpc_chttp2_hpack_compressor* c);
-void grpc_chttp2_hpack_compressor_destroy(grpc_chttp2_hpack_compressor* c);
-void grpc_chttp2_hpack_compressor_set_max_table_size(
-    grpc_chttp2_hpack_compressor* c, uint32_t max_table_size);
-void grpc_chttp2_hpack_compressor_set_max_usable_size(
-    grpc_chttp2_hpack_compressor* c, uint32_t max_table_size);
-
-struct grpc_encode_header_options {
-  uint32_t stream_id;
-  bool is_eof;
-  bool use_true_binary_metadata;
-  size_t max_frame_size;
-  grpc_transport_one_way_stats* stats;
+    void EncodeAlwaysIndexed(uint32_t* index, const grpc_slice& key,
+                             const grpc_slice& value,
+                             uint32_t transport_length);
+
+    size_t CurrentFrameSize() const;
+    void Add(grpc_slice slice);
+    uint8_t* AddTiny(size_t len);
+
+    // maximum size of a frame
+    const size_t max_frame_size_;
+    bool is_first_frame_ = true;
+    const bool use_true_binary_metadata_;
+    const bool is_end_of_stream_;
+    // output stream id
+    const uint32_t stream_id_;
+#ifndef NDEBUG
+    // have we seen a regular (non-colon-prefixed) header yet?
+    bool seen_regular_header_ = false;
+#endif
+    grpc_slice_buffer* const output_;
+    grpc_transport_one_way_stats* const stats_;
+    HPackCompressor* const compressor_;
+    FramePrefix prefix_;
+  };
+
+ private:
+  static constexpr size_t kNumFilterValues = 64;
+
+  void AddKeyWithIndex(grpc_slice_refcount* key_ref, uint32_t new_index,
+                       uint32_t key_hash);
+  void AddElemWithIndex(grpc_mdelem elem, uint32_t new_index,
+                        uint32_t elem_hash, uint32_t key_hash);
+  void AddElem(grpc_mdelem elem, size_t elem_size, uint32_t elem_hash,
+               uint32_t key_hash);
+  void AddKey(grpc_mdelem elem, size_t elem_size, uint32_t key_hash);
+
+  // maximum number of bytes we'll use for the decode table (to guard against
+  // peers ooming us by setting decode table size high)
+  uint32_t max_usable_size_ = hpack_constants::kInitialTableSize;
+  // if non-zero, advertise to the decoder that we'll start using a table
+  // of this size
+  bool advertise_table_size_change_ = false;
+  HPackEncoderTable table_;
+
+  // filter tables for elems: this tables provides an approximate
+  // popularity count for particular hashes, and are used to determine whether
+  // a new literal should be added to the compression table or not.
+  // They track a single integer that counts how often a particular value has
+  // been seen. When that count reaches max (255), all values are halved.
+  PopularityCount<kNumFilterValues> filter_elems_;
+
+  class KeyElem {
+   public:
+    class Stored {
+     public:
+      Stored() : elem_(GRPC_MDNULL) {}
+      explicit Stored(grpc_mdelem elem) : elem_(GRPC_MDELEM_REF(elem)) {}
+      Stored(const Stored& other) : elem_(GRPC_MDELEM_REF(other.elem_)) {}
+      Stored& operator=(Stored other) {
+        std::swap(elem_, other.elem_);
+        return *this;
+      }
+      ~Stored() { GRPC_MDELEM_UNREF(elem_); }
+
+      const grpc_mdelem& elem() const { return elem_; }
+
+      bool operator==(const Stored& other) const noexcept {
+        return elem_.payload == other.elem_.payload;
+      }
+
+     private:
+      grpc_mdelem elem_;
+    };
+
+    KeyElem(grpc_mdelem elem, uint32_t hash) : elem_(elem), hash_(hash) {}
+    KeyElem(const KeyElem&);
+    KeyElem& operator=(const KeyElem&);
+
+    uint32_t hash() const {
+      // TODO(ctiller): unify this with what's in the cc file when we move this
+      // code to c++
+      return hash_ >> 6;
+    }
+
+    Stored stored() const { return Stored(elem_); }
+
+    bool operator==(const Stored& stored) const noexcept {
+      return elem_.payload == stored.elem().payload;
+    }
+
+   private:
+    grpc_mdelem elem_;
+    uint32_t hash_;
+  };
+
+  class KeySliceRef {
+   public:
+    using Stored = RefCountedPtr<grpc_slice_refcount>;
+
+    KeySliceRef(grpc_slice_refcount* ref, uint32_t hash)
+        : ref_(ref), hash_(hash) {}
+    KeySliceRef(const KeySliceRef&) = delete;
+    KeySliceRef& operator=(const KeySliceRef&) = delete;
+
+    uint32_t hash() const {
+      // TODO(ctiller): unify this with what's in the cc file when we move this
+      // code to c++
+      return hash_ >> 6;
+    }
+
+    Stored stored() const {
+      ref_->Ref();
+      return Stored(ref_);
+    }
+
+    bool operator==(const Stored& stored) const noexcept {
+      return ref_ == stored.get();
+    }
+
+   private:
+    grpc_slice_refcount* ref_;
+    uint32_t hash_;
+  };
+
+  // entry tables for keys & elems: these tables track values that have been
+  // seen and *may* be in the decompressor table
+  HPackEncoderIndex<KeyElem, kNumFilterValues> elem_index_;
+  HPackEncoderIndex<KeySliceRef, kNumFilterValues> key_index_;
+  // Index into table_ for the te:trailers metadata element
+  uint32_t te_index_ = 0;
+  // Index into table_ for the user-agent metadata element
+  uint32_t user_agent_index_ = 0;
+  // The user-agent string referred to by user_agent_index_
+  Slice user_agent_;
 };
-void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor* c,
-                               grpc_mdelem** extra_headers,
-                               size_t extra_headers_size,
-                               grpc_metadata_batch* metadata,
-                               const grpc_encode_header_options* options,
-                               grpc_slice_buffer* outbuf);
+
+}  // namespace grpc_core
 
 #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_H */

data/src/core/ext/transport/chttp2/transport/hpack_encoder_index.h

@@ -0,0 +1,107 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_INDEX_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_INDEX_H
+
+#include <grpc/support/port_platform.h>
+
+#include "absl/types/optional.h"
+
+namespace grpc_core {
+
+// A fixed size mapping of a key to a chronologically ordered index
+template <typename Key, size_t kNumEntries>
+class HPackEncoderIndex {
+ public:
+  using Index = uint32_t;
+
+  HPackEncoderIndex() : entries_{} {}
+
+  // If key exists in the table, update it to a new index.
+  // If it does not and there is an empty slot, add it to the index.
+  // Finally, if it does not and there is no empty slot, evict the oldest
+  // conflicting member.
+  void Insert(const Key& key, Index new_index) {
+    auto* const cuckoo_first = first_slot(key);
+    if (cuckoo_first->UpdateOrAdd(key, new_index)) return;
+    auto* const cuckoo_second = second_slot(key);
+    if (cuckoo_second->UpdateOrAdd(key, new_index)) return;
+    auto* const clobber = Older(cuckoo_first, cuckoo_second);
+    clobber->key = key.stored();
+    clobber->index = new_index;
+  }
+
+  // Lookup key and return its index, or return empty if it's not in this table.
+  absl::optional<Index> Lookup(const Key& key) {
+    auto* const cuckoo_first = first_slot(key);
+    if (key == cuckoo_first->key) return cuckoo_first->index;
+    auto* const cuckoo_second = second_slot(key);
+    if (key == cuckoo_second->key) return cuckoo_second->index;
+    return {};
+  }
+
+ private:
+  using StoredKey = typename Key::Stored;
+
+  // One entry in the index
+  struct Entry {
+    Entry() : key{}, index{} {};
+
+    StoredKey key;
+    Index index;
+
+    // Update this entry if it matches key, otherwise if it's empty add it.
+    // If neither happens, return false.
+    bool UpdateOrAdd(const Key& new_key, Index new_index) {
+      if (new_key == key) {
+        index = new_index;
+        return true;
+      } else if (key == StoredKey()) {
+        key = new_key.stored();
+        index = new_index;
+        return true;
+      } else {
+        return false;
+      }
+    }
+  };
+
+  static Entry* Older(Entry* a, Entry* b) {
+    if (a->index < b->index) {
+      return a;
+    } else {
+      return b;
+    }
+  }
+
+  // Return the first slot in which key could be stored.
+  Entry* first_slot(const Key& key) {
+    return &entries_[key.hash() % kNumEntries];
+  }
+
+  // Return the second slot in which key could be stored.
+  Entry* second_slot(const Key& key) {
+    return &entries_[(key.hash() / kNumEntries) % kNumEntries];
+  }
+
+  // Fixed size entry map.
+  // We store each key/value pair in two slots based on it's hash value.
+  // They can be evicted individually.
+  Entry entries_[kNumEntries];
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_INDEX_H

data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc

@@ -0,0 +1,86 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/ext/transport/chttp2/transport/hpack_encoder_table.h"
+
+#include <grpc/support/log.h>
+
+namespace grpc_core {
+
+uint32_t HPackEncoderTable::AllocateIndex(size_t element_size) {
+  uint32_t new_index = tail_remote_index_ + table_elems_ + 1;
+  GPR_DEBUG_ASSERT(element_size < 65536);
+
+  if (element_size > max_table_size_) {
+    while (table_size_ > 0) {
+      EvictOne();
+    }
+    return 0;
+  }
+
+  // Reserve space for this element in the remote table: if this overflows
+  // the current table, drop elements until it fits, matching the decompressor
+  // algorithm.
+  while (table_size_ + element_size > max_table_size_) {
+    EvictOne();
+  }
+  GPR_ASSERT(table_elems_ < elem_size_.size());
+  elem_size_[new_index % elem_size_.size()] =
+      static_cast<uint16_t>(element_size);
+  table_size_ += element_size;
+  table_elems_++;
+
+  return new_index;
+}
+
+bool HPackEncoderTable::SetMaxSize(uint32_t max_table_size) {
+  if (max_table_size == max_table_size_) {
+    return false;
+  }
+  while (table_size_ > 0 && table_size_ > max_table_size) {
+    EvictOne();
+  }
+  max_table_size_ = max_table_size;
+  const size_t max_table_elems =
+      hpack_constants::EntriesForBytes(max_table_size);
+  // TODO(ctiller): integrate with ResourceQuota to rebuild smaller when we can.
+  if (max_table_elems > elem_size_.size()) {
+    Rebuild(std::max(max_table_elems, 2 * elem_size_.size()));
+  }
+  return true;
+}
+
+void HPackEncoderTable::EvictOne() {
+  tail_remote_index_++;
+  GPR_ASSERT(tail_remote_index_ > 0);
+  GPR_ASSERT(table_elems_ > 0);
+  auto removing_size = elem_size_[tail_remote_index_ % elem_size_.size()];
+  GPR_ASSERT(table_size_ >= removing_size);
+  table_size_ -= removing_size;
+  table_elems_--;
+}
+
+void HPackEncoderTable::Rebuild(uint32_t capacity) {
+  decltype(elem_size_) new_elem_size(capacity);
+  GPR_ASSERT(table_elems_ <= capacity);
+  for (uint32_t i = 0; i < table_elems_; i++) {
+    uint32_t ofs = tail_remote_index_ + i + 1;
+    new_elem_size[ofs % capacity] = elem_size_[ofs % elem_size_.size()];
+  }
+  elem_size_.swap(new_elem_size);
+}
+
+}  // namespace grpc_core

data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.h

@@ -0,0 +1,69 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_TABLE_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_TABLE_H
+
+#include <grpc/support/port_platform.h>
+
+#include "absl/container/inlined_vector.h"
+
+#include "src/core/ext/transport/chttp2/transport/hpack_constants.h"
+
+namespace grpc_core {
+
+// Tracks the values available in the remote HPACK header table, and their
+// sizes.
+class HPackEncoderTable {
+ public:
+  HPackEncoderTable() : elem_size_(hpack_constants::kInitialTableEntries) {}
+
+  // Reserve space in table for the new element, evict entries if needed.
+  // Return the new index of the element. Return 0 to indicate not adding to
+  // table.
+  uint32_t AllocateIndex(size_t element_size);
+  // Set the maximum table size. Return true if it changed.
+  bool SetMaxSize(uint32_t max_table_size);
+  // Get the current max table size
+  uint32_t max_size() const { return max_table_size_; }
+  // Get the current table size
+  uint32_t test_only_table_size() const { return table_size_; }
+
+  // Convert an element index into a dynamic index
+  uint32_t DynamicIndex(uint32_t index) const {
+    return 1 + hpack_constants::kLastStaticEntry + tail_remote_index_ +
+           table_elems_ - index;
+  }
+  // Check if an element index is convertable to a dynamic index
+  bool ConvertableToDynamicIndex(uint32_t index) const {
+    return index > tail_remote_index_;
+  }
+
+ private:
+  void EvictOne();
+  void Rebuild(uint32_t capacity);
+
+  // one before the lowest usable table index
+  uint32_t tail_remote_index_ = 0;
+  uint32_t max_table_size_ = hpack_constants::kInitialTableSize;
+  uint32_t table_elems_ = 0;
+  uint32_t table_size_ = 0;
+  // The size of each element in the HPACK table.
+  absl::InlinedVector<uint16_t, hpack_constants::kInitialTableEntries>
+      elem_size_;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_TABLE_H