@nxtedition/rocksdb 8.1.17 → 8.2.0-alpha.2

package/deps/rocksdb/rocksdb/db/internal_stats.h

@@ -104,15 +104,20 @@ class InternalStats {
   static const std::map<LevelStatType, LevelStat> compaction_level_stats;
 
   enum InternalCFStatsType {
-    L0_FILE_COUNT_LIMIT_SLOWDOWNS,
-    LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS,
+    MEMTABLE_LIMIT_DELAYS,
     MEMTABLE_LIMIT_STOPS,
-    MEMTABLE_LIMIT_SLOWDOWNS,
+    L0_FILE_COUNT_LIMIT_DELAYS,
     L0_FILE_COUNT_LIMIT_STOPS,
-    LOCKED_L0_FILE_COUNT_LIMIT_STOPS,
-    PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS,
+    PENDING_COMPACTION_BYTES_LIMIT_DELAYS,
     PENDING_COMPACTION_BYTES_LIMIT_STOPS,
+    // Write slowdown caused by l0 file count limit while there is ongoing L0
+    // compaction
+    L0_FILE_COUNT_LIMIT_DELAYS_WITH_ONGOING_COMPACTION,
+    // Write stop caused by l0 file count limit while there is ongoing L0
+    // compaction
+    L0_FILE_COUNT_LIMIT_STOPS_WITH_ONGOING_COMPACTION,
     WRITE_STALLS_ENUM_MAX,
+    // End of all write stall stats
     BYTES_FLUSHED,
     BYTES_INGESTED_ADD_FILE,
     INGESTED_NUM_FILES_TOTAL,
@@ -129,7 +134,18 @@ class InternalStats {
     kIntStatsWriteDoneByOther,
     kIntStatsWriteDoneBySelf,
     kIntStatsWriteWithWal,
+    // TODO(hx235): Currently `kIntStatsWriteStallMicros` only measures
+    // "delayed" time of CF-scope write stalls, not including the "stopped" time
+    // nor any DB-scope write stalls (e.g, ones triggered by
+    // `WriteBufferManager`).
+    //
+    // However, the word "write stall" includes both "delayed" and "stopped"
+    // (see `WriteStallCondition`) and DB-scope writes stalls (see
+    // `WriteStallCause`).
+    //
+    // So we should improve, rename or clarify it
     kIntStatsWriteStallMicros,
+    kIntStatsWriteBufferManagerLimitStopsCounts,
     kIntStatsNumMax,
   };
 
@@ -599,6 +615,10 @@ class InternalStats {
  private:
   void DumpDBMapStats(std::map<std::string, std::string>* db_stats);
   void DumpDBStats(std::string* value);
+
+  void DumpDBMapStatsWriteStall(std::map<std::string, std::string>* value);
+  void DumpDBStatsWriteStall(std::string* value);
+
   void DumpCFMapStats(std::map<std::string, std::string>* cf_stats);
   void DumpCFMapStats(
       const VersionStorageInfo* vstorage,
@@ -606,7 +626,6 @@ class InternalStats {
       CompactionStats* compaction_stats_sum);
   void DumpCFMapStatsByPriority(
       std::map<int, std::map<LevelStatType, double>>* priorities_stats);
-  void DumpCFMapStatsIOStalls(std::map<std::string, std::string>* cf_stats);
   void DumpCFStats(std::string* value);
   // if is_periodic = true, it is an internal call by RocksDB periodically to
   // dump the status.
@@ -615,6 +634,10 @@ class InternalStats {
   // dump the status.
   void DumpCFFileHistogram(std::string* value);
 
+  void DumpCFMapStatsWriteStall(std::map<std::string, std::string>* value);
+  void DumpCFStatsWriteStall(std::string* value,
+                             uint64_t* total_stall_count = nullptr);
+
   Cache* GetBlockCacheForStats();
   Cache* GetBlobCacheForStats();
 
@@ -648,7 +671,7 @@ class InternalStats {
     // ColumnFamily-level stats
     CompactionStats comp_stats;
     uint64_t ingest_bytes_flush;  // Bytes written to L0 (Flush)
-    uint64_t stall_count;         // Stall count
+    uint64_t stall_count;         // Total counts of CF-scope write stalls
     // Stats from compaction jobs - bytes written, bytes read, duration.
     uint64_t compact_bytes_write;
     uint64_t compact_bytes_read;
@@ -743,9 +766,15 @@ class InternalStats {
   bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix);
   bool HandleCFFileHistogram(std::string* value, Slice suffix);
   bool HandleCFStatsPeriodic(std::string* value, Slice suffix);
+  bool HandleCFWriteStallStats(std::string* value, Slice suffix);
+  bool HandleCFWriteStallStatsMap(std::map<std::string, std::string>* values,
+                                  Slice suffix);
   bool HandleDBMapStats(std::map<std::string, std::string>* compaction_stats,
                         Slice suffix);
   bool HandleDBStats(std::string* value, Slice suffix);
+  bool HandleDBWriteStallStats(std::string* value, Slice suffix);
+  bool HandleDBWriteStallStatsMap(std::map<std::string, std::string>* values,
+                                  Slice suffix);
   bool HandleSsTables(std::string* value, Slice suffix);
   bool HandleAggregatedTableProperties(std::string* value, Slice suffix);
   bool HandleAggregatedTablePropertiesAtLevel(std::string* value, Slice suffix);

package/deps/rocksdb/rocksdb/db/memtable.cc

@@ -1078,7 +1078,7 @@ static bool SaveValue(void* arg, const char* entry) {
                 *(s->value) = std::move(result);
               } else {
                 assert(s->columns);
-                s->columns->SetPlainValue(result);
+                s->columns->SetPlainValue(std::move(result));
               }
             }
           }
@@ -1152,7 +1152,7 @@ static bool SaveValue(void* arg, const char* entry) {
                 /* op_failure_scope */ nullptr);
 
             if (s->status->ok()) {
-              *(s->status) = s->columns->SetWideColumnValue(result);
+              *(s->status) = s->columns->SetWideColumnValue(std::move(result));
             }
           }
         } else if (s->value) {
@@ -1200,7 +1200,7 @@ static bool SaveValue(void* arg, const char* entry) {
                 *(s->value) = std::move(result);
               } else {
                 assert(s->columns);
-                s->columns->SetPlainValue(result);
+                s->columns->SetPlainValue(std::move(result));
               }
             }
           } else {
@@ -1230,6 +1230,8 @@ static bool SaveValue(void* arg, const char* entry) {
         *(s->merge_in_progress) = true;
         merge_context->PushOperand(
             v, s->inplace_update_support == false /* operand_pinned */);
+        PERF_COUNTER_ADD(internal_merge_point_lookup_count, 1);
+
         if (s->do_merge && merge_operator->ShouldMerge(
                                merge_context->GetOperandsDirectionBackward())) {
           if (s->value || s->columns) {
@@ -1249,7 +1251,7 @@ static bool SaveValue(void* arg, const char* entry) {
                 *(s->value) = std::move(result);
               } else {
                 assert(s->columns);
-                s->columns->SetPlainValue(result);
+                s->columns->SetPlainValue(std::move(result));
               }
             }
           }

package/deps/rocksdb/rocksdb/db/merge_helper.cc

@@ -231,6 +231,10 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
       s = Status::ShutdownInProgress();
       return s;
     }
+    // Skip range tombstones emitted by the compaction iterator.
+    if (iter->IsDeleteRangeSentinelKey()) {
+      continue;
+    }
 
     ParsedInternalKey ikey;
     assert(keys_.size() == merge_context_.GetNumOperands());

package/deps/rocksdb/rocksdb/db/perf_context_test.cc

@@ -964,6 +964,157 @@ TEST_F(PerfContextTest, CPUTimer) {
     ASSERT_EQ(count, get_perf_context()->iter_seek_cpu_nanos);
   }
 }
+
+TEST_F(PerfContextTest, MergeOperandCount) {
+  ASSERT_OK(DestroyDB(kDbName, Options()));
+
+  DB* db = nullptr;
+  Options options;
+  options.create_if_missing = true;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+
+  ASSERT_OK(DB::Open(options, kDbName, &db));
+  std::unique_ptr<DB> db_guard(db);
+
+  constexpr size_t num_keys = 3;
+  const std::string key_prefix("key");
+  const std::string value_prefix("value");
+
+  std::vector<std::string> keys;
+  keys.reserve(num_keys);
+
+  for (size_t i = 0; i < num_keys; ++i) {
+    keys.emplace_back(key_prefix + std::to_string(i));
+  }
+
+  // Write three keys with one Put each followed by 1, 2, and 3
+  // Merge operations respectively.
+  constexpr size_t total_merges = num_keys * (num_keys + 1) / 2;
+
+  std::vector<ManagedSnapshot> snapshots;
+  snapshots.reserve(total_merges);
+
+  for (size_t i = 0; i < num_keys; ++i) {
+    const std::string suffix = std::to_string(i);
+    const std::string value = value_prefix + suffix;
+
+    ASSERT_OK(db->Put(WriteOptions(), keys[i], value));
+
+    for (size_t j = 0; j <= i; ++j) {
+      // Take a snapshot before each Merge so they are preserved and not
+      // collapsed during flush.
+      snapshots.emplace_back(db);
+
+      ASSERT_OK(db->Merge(WriteOptions(), keys[i], value + std::to_string(j)));
+    }
+  }
+
+  auto verify = [&]() {
+    get_perf_context()->Reset();
+
+    for (size_t i = 0; i < num_keys; ++i) {
+      // Get
+      {
+        PinnableSlice result;
+        ASSERT_OK(db->Get(ReadOptions(), db->DefaultColumnFamily(), keys[i],
+                          &result));
+        ASSERT_EQ(get_perf_context()->internal_merge_point_lookup_count, i + 1);
+
+        get_perf_context()->Reset();
+      }
+
+      // GetEntity
+      {
+        PinnableWideColumns result;
+        ASSERT_OK(db->GetEntity(ReadOptions(), db->DefaultColumnFamily(),
+                                keys[i], &result));
+        ASSERT_EQ(get_perf_context()->internal_merge_point_lookup_count, i + 1);
+
+        get_perf_context()->Reset();
+      }
+    }
+
+    {
+      std::vector<Slice> key_slices;
+      key_slices.reserve(num_keys);
+
+      for (size_t i = 0; i < num_keys; ++i) {
+        key_slices.emplace_back(keys[i]);
+      }
+
+      // MultiGet
+      {
+        std::vector<PinnableSlice> results(num_keys);
+        std::vector<Status> statuses(num_keys);
+
+        db->MultiGet(ReadOptions(), db->DefaultColumnFamily(), num_keys,
+                     &key_slices[0], &results[0], &statuses[0]);
+
+        for (size_t i = 0; i < num_keys; ++i) {
+          ASSERT_OK(statuses[i]);
+        }
+
+        ASSERT_EQ(get_perf_context()->internal_merge_point_lookup_count,
+                  total_merges);
+
+        get_perf_context()->Reset();
+      }
+
+      // MultiGetEntity
+      {
+        std::vector<PinnableWideColumns> results(num_keys);
+        std::vector<Status> statuses(num_keys);
+
+        db->MultiGetEntity(ReadOptions(), db->DefaultColumnFamily(), num_keys,
+                           &key_slices[0], &results[0], &statuses[0]);
+
+        for (size_t i = 0; i < num_keys; ++i) {
+          ASSERT_OK(statuses[i]);
+        }
+
+        ASSERT_EQ(get_perf_context()->internal_merge_point_lookup_count,
+                  total_merges);
+
+        get_perf_context()->Reset();
+      }
+    }
+
+    std::unique_ptr<Iterator> it(db->NewIterator(ReadOptions()));
+
+    // Forward iteration
+    {
+      size_t i = 0;
+
+      for (it->SeekToFirst(); it->Valid(); it->Next(), ++i) {
+        ASSERT_EQ(it->key(), keys[i]);
+        ASSERT_EQ(get_perf_context()->internal_merge_count, i + 1);
+
+        get_perf_context()->Reset();
+      }
+    }
+
+    // Backward iteration
+    {
+      size_t i = num_keys - 1;
+
+      for (it->SeekToLast(); it->Valid(); it->Prev(), --i) {
+        ASSERT_EQ(it->key(), keys[i]);
+        ASSERT_EQ(get_perf_context()->internal_merge_count, i + 1);
+
+        get_perf_context()->Reset();
+      }
+    }
+  };
+
+  // Verify counters when reading from memtable
+  verify();
+
+  // Verify counters when reading from table files
+  db->Flush(FlushOptions());
+
+  verify();
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb/db/range_del_aggregator.cc

@@ -30,12 +30,15 @@ TruncatedRangeDelIterator::TruncatedRangeDelIterator(
       icmp_(icmp),
       smallest_ikey_(smallest),
       largest_ikey_(largest) {
+  // Set up bounds such that range tombstones from this iterator are
+  // truncated to range [smallest_, largest_).
   if (smallest != nullptr) {
     pinned_bounds_.emplace_back();
     auto& parsed_smallest = pinned_bounds_.back();
     Status pik_status = ParseInternalKey(smallest->Encode(), &parsed_smallest,
                                          false /* log_err_key */);  // TODO
     pik_status.PermitUncheckedError();
+    parsed_smallest.type = kTypeMaxValid;
     assert(pik_status.ok());
     smallest_ = &parsed_smallest;
   }
@@ -63,6 +66,8 @@ TruncatedRangeDelIterator::TruncatedRangeDelIterator(
       //
       // Therefore, we will never truncate a range tombstone at largest, so we
       // can leave it unchanged.
+      // TODO: maybe use kMaxValid here to ensure range tombstone having
+      //  distinct key from point keys.
     } else {
       // The same user key may straddle two sstable boundaries. To ensure that
       // the truncated end key can cover the largest key in this sstable, reduce
@@ -70,7 +75,7 @@ TruncatedRangeDelIterator::TruncatedRangeDelIterator(
       parsed_largest.sequence -= 1;
       // This line is not needed for correctness, but it ensures that the
       // truncated end key is not covering keys from the next SST file.
-      parsed_largest.type = kValueTypeForSeek;
+      parsed_largest.type = kTypeMaxValid;
     }
     largest_ = &parsed_largest;
   }
@@ -101,6 +106,24 @@ void TruncatedRangeDelIterator::Seek(const Slice& target) {
   iter_->Seek(target);
 }
 
+void TruncatedRangeDelIterator::SeekInternalKey(const Slice& target) {
+  if (largest_ && icmp_->Compare(*largest_, target) <= 0) {
+    iter_->Invalidate();
+    return;
+  }
+  if (smallest_ && icmp_->Compare(target, *smallest_) < 0) {
+    // Since target < smallest, target < largest_.
+    // This seek must land on a range tombstone where end_key() > target,
+    // so there is no need to check again.
+    iter_->Seek(smallest_->user_key);
+  } else {
+    iter_->Seek(ExtractUserKey(target));
+    while (Valid() && icmp_->Compare(end_key(), target) <= 0) {
+      Next();
+    }
+  }
+}
+
 // NOTE: target is a user key, with timestamp if enabled.
 void TruncatedRangeDelIterator::SeekForPrev(const Slice& target) {
   if (smallest_ != nullptr &&
@@ -393,21 +416,20 @@ bool CompactionRangeDelAggregator::ShouldDelete(const ParsedInternalKey& parsed,
 namespace {
 
 // Produce a sorted (by start internal key) stream of range tombstones from
-// `children`. lower_bound and upper_bound on user key can be
+// `children`. lower_bound and upper_bound on internal key can be
 // optionally specified. Range tombstones that ends before lower_bound or starts
 // after upper_bound are excluded.
 // If user-defined timestamp is enabled, lower_bound and upper_bound should
-// contain timestamp, but comparison is done ignoring timestamps.
+// contain timestamp.
 class TruncatedRangeDelMergingIter : public InternalIterator {
  public:
   TruncatedRangeDelMergingIter(
       const InternalKeyComparator* icmp, const Slice* lower_bound,
-      const Slice* upper_bound, bool upper_bound_inclusive,
+      const Slice* upper_bound,
       const std::vector<std::unique_ptr<TruncatedRangeDelIterator>>& children)
       : icmp_(icmp),
         lower_bound_(lower_bound),
         upper_bound_(upper_bound),
-        upper_bound_inclusive_(upper_bound_inclusive),
         heap_(StartKeyMinComparator(icmp)),
         ts_sz_(icmp_->user_comparator()->timestamp_size()) {
     for (auto& child : children) {
@@ -420,7 +442,7 @@ class TruncatedRangeDelMergingIter : public InternalIterator {
   }
 
   bool Valid() const override {
-    return !heap_.empty() && BeforeEndKey(heap_.top());
+    return !heap_.empty() && !AfterEndKey(heap_.top());
   }
   Status status() const override { return Status::OK(); }
 
@@ -428,7 +450,13 @@ class TruncatedRangeDelMergingIter : public InternalIterator {
     heap_.clear();
     for (auto& child : children_) {
       if (lower_bound_ != nullptr) {
-        child->Seek(*lower_bound_);
+        child->Seek(ExtractUserKey(*lower_bound_));
+        // Since the above `Seek()` operates on a user key while `lower_bound_`
+        // is an internal key, we may need to advance `child` farther for it to
+        // be in bounds.
+        while (child->Valid() && BeforeStartKey(child)) {
+          child->InternalNext();
+        }
       } else {
         child->SeekToFirst();
       }
@@ -481,19 +509,23 @@ class TruncatedRangeDelMergingIter : public InternalIterator {
   void SeekToLast() override { assert(false); }
 
  private:
-  bool BeforeEndKey(const TruncatedRangeDelIterator* iter) const {
+  bool BeforeStartKey(const TruncatedRangeDelIterator* iter) const {
+    if (lower_bound_ == nullptr) {
+      return false;
+    }
+    return icmp_->Compare(iter->end_key(), *lower_bound_) <= 0;
+  }
+
+  bool AfterEndKey(const TruncatedRangeDelIterator* iter) const {
     if (upper_bound_ == nullptr) {
-      return true;
+      return false;
     }
-    int cmp = icmp_->user_comparator()->CompareWithoutTimestamp(
-        iter->start_key().user_key, *upper_bound_);
-    return upper_bound_inclusive_ ? cmp <= 0 : cmp < 0;
+    return icmp_->Compare(iter->start_key(), *upper_bound_) > 0;
   }
 
   const InternalKeyComparator* icmp_;
   const Slice* lower_bound_;
   const Slice* upper_bound_;
-  bool upper_bound_inclusive_;
   BinaryHeap<TruncatedRangeDelIterator*, StartKeyMinComparator> heap_;
   std::vector<TruncatedRangeDelIterator*> children_;
 
@@ -506,11 +538,10 @@ class TruncatedRangeDelMergingIter : public InternalIterator {
 
 std::unique_ptr<FragmentedRangeTombstoneIterator>
 CompactionRangeDelAggregator::NewIterator(const Slice* lower_bound,
-                                          const Slice* upper_bound,
-                                          bool upper_bound_inclusive) {
+                                          const Slice* upper_bound) {
   InvalidateRangeDelMapPositions();
   auto merging_iter = std::make_unique<TruncatedRangeDelMergingIter>(
-      icmp_, lower_bound, upper_bound, upper_bound_inclusive, parent_iters_);
+      icmp_, lower_bound, upper_bound, parent_iters_);
 
   auto fragmented_tombstone_list =
       std::make_shared<FragmentedRangeTombstoneList>(

package/deps/rocksdb/rocksdb/db/range_del_aggregator.h

@@ -49,6 +49,9 @@ class TruncatedRangeDelIterator {
   // REQUIRES: target is a user key.
   void Seek(const Slice& target);
 
+  // Seeks to the first range tombstone with end_key() > target.
+  void SeekInternalKey(const Slice& target);
+
   // Seeks to the tombstone with the highest visible sequence number that covers
   // target (a user key). If no such tombstone exists, the position will be at
   // the latest tombstone that starts before target.
@@ -452,16 +455,15 @@ class CompactionRangeDelAggregator : public RangeDelAggregator {
   }
 
   // Creates an iterator over all the range tombstones in the aggregator, for
-  // use in compaction. Nullptr arguments indicate that the iterator range is
-  // unbounded.
-  // NOTE: the boundaries are used for optimization purposes to reduce the
-  // number of tombstones that are passed to the fragmenter; they do not
-  // guarantee that the resulting iterator only contains range tombstones that
-  // cover keys in the provided range. If required, these bounds must be
+  // use in compaction.
+  //
+  // NOTE: the internal key boundaries are used for optimization purposes to
+  // reduce the number of tombstones that are passed to the fragmenter; they do
+  // not guarantee that the resulting iterator only contains range tombstones
+  // that cover keys in the provided range. If required, these bounds must be
   // enforced during iteration.
   std::unique_ptr<FragmentedRangeTombstoneIterator> NewIterator(
-      const Slice* lower_bound = nullptr, const Slice* upper_bound = nullptr,
-      bool upper_bound_inclusive = false);
+      const Slice* lower_bound = nullptr, const Slice* upper_bound = nullptr);
 
  private:
   std::vector<std::unique_ptr<TruncatedRangeDelIterator>> parent_iters_;