@nxtedition/rocksdb 8.2.8 → 9.0.1

package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.cc

@@ -11,14 +11,34 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-uint64_t SeqnoToTimeMapping::GetOldestApproximateTime(
-    const SequenceNumber seqno) const {
+SeqnoToTimeMapping::pair_const_iterator SeqnoToTimeMapping::FindGreaterTime(
+    uint64_t time) const {
+  return std::upper_bound(pairs_.cbegin(), pairs_.cend(),
+                          SeqnoTimePair{0, time}, SeqnoTimePair::TimeLess);
+}
+
+SeqnoToTimeMapping::pair_const_iterator SeqnoToTimeMapping::FindGreaterEqSeqno(
+    SequenceNumber seqno) const {
+  return std::lower_bound(pairs_.cbegin(), pairs_.cend(),
+                          SeqnoTimePair{seqno, 0}, SeqnoTimePair::SeqnoLess);
+}
+
+SeqnoToTimeMapping::pair_const_iterator SeqnoToTimeMapping::FindGreaterSeqno(
+    SequenceNumber seqno) const {
+  return std::upper_bound(pairs_.cbegin(), pairs_.cend(),
+                          SeqnoTimePair{seqno, 0}, SeqnoTimePair::SeqnoLess);
+}
+
+uint64_t SeqnoToTimeMapping::GetProximalTimeBeforeSeqno(
+    SequenceNumber seqno) const {
   assert(is_sorted_);
-  auto it = std::upper_bound(seqno_time_mapping_.begin(),
-                             seqno_time_mapping_.end(), seqno);
-  if (it == seqno_time_mapping_.begin()) {
-    return 0;
+  // Find the last entry with a seqno strictly less than the given seqno.
+  // First, find the first entry >= the given seqno (or end)
+  auto it = FindGreaterEqSeqno(seqno);
+  if (it == pairs_.cbegin()) {
+    return kUnknownTimeBeforeAll;
   }
+  // Then return data from previous.
   it--;
   return it->time;
 }
@@ -28,44 +48,47 @@ void SeqnoToTimeMapping::Add(SequenceNumber seqno, uint64_t time) {
     return;
   }
   is_sorted_ = false;
-  seqno_time_mapping_.emplace_back(seqno, time);
+  pairs_.emplace_back(seqno, time);
 }
 
 void SeqnoToTimeMapping::TruncateOldEntries(const uint64_t now) {
   assert(is_sorted_);
 
   if (max_time_duration_ == 0) {
+    // No cutoff time
+    return;
+  }
+
+  if (now < max_time_duration_) {
+    // Would under-flow
     return;
   }
 
-  const uint64_t cut_off_time =
-      now > max_time_duration_ ? now - max_time_duration_ : 0;
-  assert(cut_off_time <= now);  // no overflow
+  const uint64_t cut_off_time = now - max_time_duration_;
+  assert(cut_off_time <= now);  // no under/overflow
 
-  auto it = std::upper_bound(
-      seqno_time_mapping_.begin(), seqno_time_mapping_.end(), cut_off_time,
-      [](uint64_t target, const SeqnoTimePair& other) -> bool {
-        return target < other.time;
-      });
-  if (it == seqno_time_mapping_.begin()) {
+  auto it = FindGreaterTime(cut_off_time);
+  if (it == pairs_.cbegin()) {
     return;
   }
-  it--;
-  seqno_time_mapping_.erase(seqno_time_mapping_.begin(), it);
+  // Move back one, to the entry that would be used to return a good seqno from
+  // GetProximalSeqnoBeforeTime(cut_off_time)
+  --it;
+  // Remove everything strictly before that entry
+  pairs_.erase(pairs_.cbegin(), std::move(it));
 }
 
-SequenceNumber SeqnoToTimeMapping::GetOldestSequenceNum(uint64_t time) {
+SequenceNumber SeqnoToTimeMapping::GetProximalSeqnoBeforeTime(uint64_t time) {
   assert(is_sorted_);
 
-  auto it = std::upper_bound(
-      seqno_time_mapping_.begin(), seqno_time_mapping_.end(), time,
-      [](uint64_t target, const SeqnoTimePair& other) -> bool {
-        return target < other.time;
-      });
-  if (it == seqno_time_mapping_.begin()) {
-    return 0;
+  // Find the last entry with a time <= the given time.
+  // First, find the first entry > the given time (or end).
+  auto it = FindGreaterTime(time);
+  if (it == pairs_.cbegin()) {
+    return kUnknownSeqnoBeforeAll;
   }
-  it--;
+  // Then return data from previous.
+  --it;
   return it->seqno;
 }
 
@@ -84,15 +107,13 @@ void SeqnoToTimeMapping::Encode(std::string& dest, const SequenceNumber start,
     return;
   }
 
-  auto start_it = std::upper_bound(seqno_time_mapping_.begin(),
-                                   seqno_time_mapping_.end(), start);
-  if (start_it != seqno_time_mapping_.begin()) {
+  auto start_it = FindGreaterSeqno(start);
+  if (start_it != pairs_.begin()) {
     start_it--;
   }
 
-  auto end_it = std::upper_bound(seqno_time_mapping_.begin(),
-                                 seqno_time_mapping_.end(), end);
-  if (end_it == seqno_time_mapping_.begin()) {
+  auto end_it = FindGreaterSeqno(end);
+  if (end_it == pairs_.begin()) {
     return;
   }
   if (start_it >= end_it) {
@@ -108,7 +129,7 @@ void SeqnoToTimeMapping::Encode(std::string& dest, const SequenceNumber start,
     }
   }
   // to include the first element
-  if (start_it != seqno_time_mapping_.begin()) {
+  if (start_it != pairs_.begin()) {
     start_it--;
   }
 
@@ -166,14 +187,14 @@ void SeqnoToTimeMapping::Encode(std::string& dest, const SequenceNumber start,
   SeqnoTimePair base;
   for (auto it = start_it; it < end_it; it++) {
     assert(base < *it);
-    SeqnoTimePair val = *it - base;
+    SeqnoTimePair val = it->ComputeDelta(base);
     base = *it;
     val.Encode(dest);
   }
 }
 
-Status SeqnoToTimeMapping::Add(const std::string& seqno_time_mapping_str) {
-  Slice input(seqno_time_mapping_str);
+Status SeqnoToTimeMapping::Add(const std::string& pairs_str) {
+  Slice input(pairs_str);
   if (input.empty()) {
     return Status::OK();
   }
@@ -189,8 +210,8 @@ Status SeqnoToTimeMapping::Add(const std::string& seqno_time_mapping_str) {
     if (!s.ok()) {
       return s;
     }
-    val.Add(base);
-    seqno_time_mapping_.emplace_back(val);
+    val.ApplyDelta(base);
+    pairs_.emplace_back(val);
     base = val;
   }
   return Status::OK();
@@ -222,33 +243,58 @@ bool SeqnoToTimeMapping::Append(SequenceNumber seqno, uint64_t time) {
       return false;
     }
     if (seqno == Last().seqno) {
-      Last().time = time;
-      return true;
+      // Updating Last() would hurt GetProximalSeqnoBeforeTime() queries, so
+      // NOT doing it (for now)
+      return false;
     }
     if (time == Last().time) {
-      // new sequence has the same time as old one, no need to add new mapping
-      return false;
+      // Updating Last() here helps GetProximalSeqnoBeforeTime() queries, so
+      // doing it (for now)
+      Last().seqno = seqno;
+      return true;
     }
   }
 
-  seqno_time_mapping_.emplace_back(seqno, time);
+  pairs_.emplace_back(seqno, time);
 
-  if (seqno_time_mapping_.size() > max_capacity_) {
-    seqno_time_mapping_.pop_front();
+  if (pairs_.size() > max_capacity_) {
+    // FIXME: be smarter about how we erase to avoid data falling off the
+    // front prematurely.
+    pairs_.pop_front();
   }
   return true;
 }
 
+bool SeqnoToTimeMapping::PrePopulate(SequenceNumber from_seqno,
+                                     SequenceNumber to_seqno,
+                                     uint64_t from_time, uint64_t to_time) {
+  assert(Empty());
+  assert(from_seqno > 0);
+  assert(to_seqno > from_seqno);
+  assert(from_time > kUnknownTimeBeforeAll);
+  assert(to_time >= from_time);
+
+  // TODO: smartly limit this to max_capacity_ representative samples
+  for (auto i = from_seqno; i <= to_seqno; i++) {
+    uint64_t t = from_time + (to_time - from_time) * (i - from_seqno) /
+                                 (to_seqno - from_seqno);
+    pairs_.emplace_back(i, t);
+  }
+
+  return /*success*/ true;
+}
+
 bool SeqnoToTimeMapping::Resize(uint64_t min_time_duration,
                                 uint64_t max_time_duration) {
   uint64_t new_max_capacity =
       CalculateMaxCapacity(min_time_duration, max_time_duration);
   if (new_max_capacity == max_capacity_) {
     return false;
-  } else if (new_max_capacity < seqno_time_mapping_.size()) {
-    uint64_t delta = seqno_time_mapping_.size() - new_max_capacity;
-    seqno_time_mapping_.erase(seqno_time_mapping_.begin(),
-                              seqno_time_mapping_.begin() + delta);
+  } else if (new_max_capacity < pairs_.size()) {
+    uint64_t delta = pairs_.size() - new_max_capacity;
+    // FIXME: be smarter about how we erase to avoid data falling off the
+    // front prematurely.
+    pairs_.erase(pairs_.begin(), pairs_.begin() + delta);
   }
   max_capacity_ = new_max_capacity;
   return true;
@@ -258,16 +304,16 @@ Status SeqnoToTimeMapping::Sort() {
   if (is_sorted_) {
     return Status::OK();
   }
-  if (seqno_time_mapping_.empty()) {
+  if (pairs_.empty()) {
     is_sorted_ = true;
     return Status::OK();
   }
 
-  std::deque<SeqnoTimePair> copy = std::move(seqno_time_mapping_);
+  std::deque<SeqnoTimePair> copy = std::move(pairs_);
 
   std::sort(copy.begin(), copy.end());
 
-  seqno_time_mapping_.clear();
+  pairs_.clear();
 
   // remove seqno = 0, which may have special meaning, like zeroed out data
   while (copy.front().seqno == 0) {
@@ -285,12 +331,12 @@ Status SeqnoToTimeMapping::Sort() {
       assert(it.seqno > prev.seqno);
       // If a larger sequence number has an older time which is not useful, skip
       if (it.time > prev.time) {
-        seqno_time_mapping_.push_back(prev);
+        pairs_.push_back(prev);
         prev = it;
       }
     }
   }
-  seqno_time_mapping_.emplace_back(prev);
+  pairs_.emplace_back(prev);
 
   is_sorted_ = true;
   return Status::OK();
@@ -298,7 +344,7 @@ Status SeqnoToTimeMapping::Sort() {
 
 std::string SeqnoToTimeMapping::ToHumanString() const {
   std::string ret;
-  for (const auto& seq_time : seqno_time_mapping_) {
+  for (const auto& seq_time : pairs_) {
     AppendNumberTo(&ret, seq_time.seqno);
     ret.append("->");
     AppendNumberTo(&ret, seq_time.time);
@@ -310,13 +356,11 @@ std::string SeqnoToTimeMapping::ToHumanString() const {
 SeqnoToTimeMapping SeqnoToTimeMapping::Copy(
     SequenceNumber smallest_seqno) const {
   SeqnoToTimeMapping ret;
-  auto it = std::upper_bound(seqno_time_mapping_.begin(),
-                             seqno_time_mapping_.end(), smallest_seqno);
-  if (it != seqno_time_mapping_.begin()) {
+  auto it = FindGreaterSeqno(smallest_seqno);
+  if (it != pairs_.begin()) {
     it--;
   }
-  std::copy(it, seqno_time_mapping_.end(),
-            std::back_inserter(ret.seqno_time_mapping_));
+  std::copy(it, pairs_.end(), std::back_inserter(ret.pairs_));
   return ret;
 }
 
@@ -330,12 +374,4 @@ uint64_t SeqnoToTimeMapping::CalculateMaxCapacity(uint64_t min_time_duration,
       max_time_duration * kMaxSeqnoTimePairsPerCF / min_time_duration);
 }
 
-SeqnoToTimeMapping::SeqnoTimePair SeqnoToTimeMapping::SeqnoTimePair::operator-(
-    const SeqnoTimePair& other) const {
-  SeqnoTimePair res;
-  res.seqno = seqno - other.seqno;
-  res.time = time - other.time;
-  return res;
-}
-
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/db/seqno_to_time_mapping.h

@@ -18,20 +18,32 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-constexpr uint64_t kUnknownSeqnoTime = 0;
-
-// SeqnoToTimeMapping stores the sequence number to time mapping, so given a
-// sequence number it can estimate the oldest possible time for that sequence
-// number. For example:
-//   10 -> 100
-//   50 -> 300
-// then if a key has seqno 19, the OldestApproximateTime would be 100, for 51 it
-// would be 300.
-// As it's a sorted list, the new entry is inserted from the back. The old data
-// will be popped from the front if they're no longer used.
+constexpr uint64_t kUnknownTimeBeforeAll = 0;
+constexpr SequenceNumber kUnknownSeqnoBeforeAll = 0;
+
+// SeqnoToTimeMapping stores a sampled mapping from sequence numbers to
+// unix times (seconds since epoch). This information provides rough bounds
+// between sequence numbers and their write times, but is primarily designed
+// for getting a best lower bound on the sequence number of data written no
+// later than a specified time.
 //
-// Note: the data struct is not thread safe, both read and write need to be
-//  synchronized by caller.
+// For ease of sampling, it is assumed that the recorded time in each pair
+// comes at or after the sequence number and before the next sequence number,
+// so this example:
+//
+// Seqno: 10,       11, ... 20,       21, ... 30,       31, ...
+// Time:  ...  500      ...      600      ...      700      ...
+//
+// would be represented as
+//   10 -> 500
+//   20 -> 600
+//   30 -> 700
+//
+// In typical operation, the list is sorted, both among seqnos and among times,
+// with a bounded number of entries, but some public working states violate
+// these constraints.
+//
+// NOT thread safe - requires external synchronization.
 class SeqnoToTimeMapping {
  public:
   // Maximum number of entries can be encoded into SST. The data is delta encode
@@ -63,28 +75,33 @@ class SeqnoToTimeMapping {
     // Decode the value from input Slice and remove it from the input
     Status Decode(Slice& input);
 
-    // subtraction of 2 SeqnoTimePair
-    SeqnoTimePair operator-(const SeqnoTimePair& other) const;
-
-    // Add 2 values together
-    void Add(const SeqnoTimePair& obj) {
-      seqno += obj.seqno;
-      time += obj.time;
+    // For delta encoding
+    SeqnoTimePair ComputeDelta(const SeqnoTimePair& base) const {
+      return {seqno - base.seqno, time - base.time};
     }
 
-    // Compare SeqnoTimePair with a sequence number, used for binary search a
-    // sequence number in a list of SeqnoTimePair
-    bool operator<(const SequenceNumber& other) const { return seqno < other; }
+    // For delta decoding
+    void ApplyDelta(const SeqnoTimePair& delta_or_base) {
+      seqno += delta_or_base.seqno;
+      time += delta_or_base.time;
+    }
 
-    // Compare 2 SeqnoTimePair
+    // Ordering used for Sort()
     bool operator<(const SeqnoTimePair& other) const {
       return std::tie(seqno, time) < std::tie(other.seqno, other.time);
     }
 
-    // Check if 2 SeqnoTimePair is the same
     bool operator==(const SeqnoTimePair& other) const {
       return std::tie(seqno, time) == std::tie(other.seqno, other.time);
     }
+
+    static bool SeqnoLess(const SeqnoTimePair& a, const SeqnoTimePair& b) {
+      return a.seqno < b.seqno;
+    }
+
+    static bool TimeLess(const SeqnoTimePair& a, const SeqnoTimePair& b) {
+      return a.time < b.time;
+    }
   };
 
   // constractor of SeqnoToTimeMapping
@@ -99,20 +116,40 @@ class SeqnoToTimeMapping {
                               uint64_t max_capacity = 0)
       : max_time_duration_(max_time_duration), max_capacity_(max_capacity) {}
 
+  // Both seqno range and time range are inclusive. ... TODO
+  //
+  bool PrePopulate(SequenceNumber from_seqno, SequenceNumber to_seqno,
+                   uint64_t from_time, uint64_t to_time);
+
   // Append a new entry to the list. The new entry should be newer than the
   // existing ones. It maintains the internal sorted status.
   bool Append(SequenceNumber seqno, uint64_t time);
 
-  // Given a sequence number, estimate it's oldest time
-  uint64_t GetOldestApproximateTime(SequenceNumber seqno) const;
-
-  // Truncate the old entries based on the current time and max_time_duration_
+  // Given a sequence number, return the best (largest / newest) known time
+  // that is no later than the write time of that given sequence number.
+  // If no such specific time is known, returns kUnknownTimeBeforeAll.
+  // Using the example in the class comment above,
+  //  GetProximalTimeBeforeSeqno(10) -> kUnknownTimeBeforeAll
+  //  GetProximalTimeBeforeSeqno(11) -> 500
+  //  GetProximalTimeBeforeSeqno(20) -> 500
+  //  GetProximalTimeBeforeSeqno(21) -> 600
+  uint64_t GetProximalTimeBeforeSeqno(SequenceNumber seqno) const;
+
+  // Remove any entries not needed for GetProximalSeqnoBeforeTime queries of
+  // times older than `now - max_time_duration_`
   void TruncateOldEntries(uint64_t now);
 
-  // Given a time, return it's oldest possible sequence number
-  SequenceNumber GetOldestSequenceNum(uint64_t time);
-
-  // Encode to a binary string
+  // Given a time, return the best (largest) sequence number whose write time
+  // is no later than that given time. If no such specific sequence number is
+  // known, returns kUnknownSeqnoBeforeAll. Using the example in the class
+  // comment above,
+  //  GetProximalSeqnoBeforeTime(499) -> kUnknownSeqnoBeforeAll
+  //  GetProximalSeqnoBeforeTime(500) -> 10
+  //  GetProximalSeqnoBeforeTime(599) -> 10
+  //  GetProximalSeqnoBeforeTime(600) -> 20
+  SequenceNumber GetProximalSeqnoBeforeTime(uint64_t time);
+
+  // Encode to a binary string. start and end seqno are both inclusive.
   void Encode(std::string& des, SequenceNumber start, SequenceNumber end,
               uint64_t now,
               uint64_t output_size = kMaxSeqnoTimePairsPerSST) const;
@@ -122,10 +159,10 @@ class SeqnoToTimeMapping {
   void Add(SequenceNumber seqno, uint64_t time);
 
   // Decode and add the entries to the current obj. The list will be unsorted
-  Status Add(const std::string& seqno_time_mapping_str);
+  Status Add(const std::string& pairs_str);
 
   // Return the number of entries
-  size_t Size() const { return seqno_time_mapping_.size(); }
+  size_t Size() const { return pairs_.size(); }
 
   // Reduce the size of internal list
   bool Resize(uint64_t min_time_duration, uint64_t max_time_duration);
@@ -145,10 +182,10 @@ class SeqnoToTimeMapping {
   SeqnoToTimeMapping Copy(SequenceNumber smallest_seqno) const;
 
   // If the internal list is empty
-  bool Empty() const { return seqno_time_mapping_.empty(); }
+  bool Empty() const { return pairs_.empty(); }
 
   // clear all entries
-  void Clear() { seqno_time_mapping_.clear(); }
+  void Clear() { pairs_.clear(); }
 
   // return the string for user message
   // Note: Not efficient, okay for print
@@ -156,7 +193,7 @@ class SeqnoToTimeMapping {
 
 #ifndef NDEBUG
   const std::deque<SeqnoTimePair>& TEST_GetInternalMapping() const {
-    return seqno_time_mapping_;
+    return pairs_;
   }
 #endif
 
@@ -167,7 +204,7 @@ class SeqnoToTimeMapping {
   uint64_t max_time_duration_;
   uint64_t max_capacity_;
 
-  std::deque<SeqnoTimePair> seqno_time_mapping_;
+  std::deque<SeqnoTimePair> pairs_;
 
   bool is_sorted_ = true;
 
@@ -176,14 +213,14 @@ class SeqnoToTimeMapping {
 
   SeqnoTimePair& Last() {
     assert(!Empty());
-    return seqno_time_mapping_.back();
+    return pairs_.back();
   }
-};
 
-// for searching the sequence number from SeqnoToTimeMapping
-inline bool operator<(const SequenceNumber& seqno,
-                      const SeqnoToTimeMapping::SeqnoTimePair& other) {
-  return seqno < other.seqno;
-}
+  using pair_const_iterator =
+      std::deque<SeqnoToTimeMapping::SeqnoTimePair>::const_iterator;
+  pair_const_iterator FindGreaterTime(uint64_t time) const;
+  pair_const_iterator FindGreaterSeqno(SequenceNumber seqno) const;
+  pair_const_iterator FindGreaterEqSeqno(SequenceNumber seqno) const;
+};
 
 }  // namespace ROCKSDB_NAMESPACE