@nxtedition/rocksdb 10.0.13 → 10.0.15

package/deps/rocksdb/rocksdb/db/dbformat.h

@@ -11,6 +11,7 @@
 #include <stdio.h>
 
 #include <memory>
+#include <optional>
 #include <string>
 #include <utility>
 
@@ -67,7 +68,9 @@ enum ValueType : unsigned char {
   kTypeDeletionWithTimestamp = 0x14,
   kTypeCommitXIDAndTimestamp = 0x15,  // WAL only
   kTypeWideColumnEntity = 0x16,
-  kTypeColumnFamilyWideColumnEntity = 0x17,  // WAL only
+  kTypeColumnFamilyWideColumnEntity = 0x17,     // WAL only
+  kTypeValuePreferredSeqno = 0x18,              // Value with a unix write time
+  kTypeColumnFamilyValuePreferredSeqno = 0x19,  // WAL only
   kTypeMaxValid,    // Should be after the last valid type, only used for
                     // validation
   kMaxValue = 0x7F  // Not used for storing records.
@@ -77,11 +80,38 @@ enum ValueType : unsigned char {
 extern const ValueType kValueTypeForSeek;
 extern const ValueType kValueTypeForSeekForPrev;
 
+// A range of user keys used internally by RocksDB. Also see `Range` used by
+// public APIs.
+struct UserKeyRange {
+  // In case of user_defined timestamp, if enabled, `start` and `limit` should
+  // include user_defined timestamps.
+  Slice start;
+  Slice limit;
+
+  UserKeyRange() = default;
+  UserKeyRange(const Slice& s, const Slice& l) : start(s), limit(l) {}
+};
+
+// A range of user keys used internally by RocksDB. Also see `RangePtr` used by
+// public APIs.
+struct UserKeyRangePtr {
+  // In case of user_defined timestamp, if enabled, `start` and `limit` should
+  // point to key with timestamp part.
+  // An optional range start, if missing, indicating a start before all keys.
+  std::optional<Slice> start;
+  // An optional range end, if missing, indicating an end after all keys.
+  std::optional<Slice> limit;
+
+  UserKeyRangePtr(const std::optional<Slice>& s, const std::optional<Slice>& l)
+      : start(s), limit(l) {}
+};
+
 // Checks whether a type is an inline value type
 // (i.e. a type used in memtable skiplist and sst file datablock).
 inline bool IsValueType(ValueType t) {
   return t <= kTypeMerge || kTypeSingleDeletion == t || kTypeBlobIndex == t ||
-         kTypeDeletionWithTimestamp == t || kTypeWideColumnEntity == t;
+         kTypeDeletionWithTimestamp == t || kTypeWideColumnEntity == t ||
+         kTypeValuePreferredSeqno == t;
 }
 
 // Checks whether a type is from user operation
@@ -118,7 +148,8 @@ struct ParsedInternalKey {
   // u contains timestamp if user timestamp feature is enabled.
   ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
       : user_key(u), sequence(seq), type(t) {}
-  std::string DebugString(bool log_err_key, bool hex) const;
+  std::string DebugString(bool log_err_key, bool hex,
+                          const Comparator* ucmp = nullptr) const;
 
   void clear() {
     user_key.clear();
@@ -165,6 +196,9 @@ inline void UnPackSequenceAndType(uint64_t packed, uint64_t* seq,
   // assert(IsExtendedValueType(*t));
 }
 
+const uint64_t kRangeTombstoneSentinel =
+    PackSequenceAndType(kMaxSequenceNumber, kTypeRangeDeletion);
+
 EntryType GetEntryType(ValueType value_type);
 
 // Append the serialization of "key" to *result.
@@ -184,6 +218,15 @@ void AppendInternalKeyWithDifferentTimestamp(std::string* result,
                                              const ParsedInternalKey& key,
                                              const Slice& ts);
 
+// Append the user key to *result, replacing the original timestamp with
+// argument ts.
+//
+// input [user key]:       <user_provided_key | original_ts>
+// output before: empty
+// output after:           <user_provided_key | ts>
+void AppendUserKeyWithDifferentTimestamp(std::string* result, const Slice& key,
+                                         const Slice& ts);
+
 // Serialized internal key consists of user key followed by footer.
 // This function appends the footer to *result, assuming that *result already
 // contains the user key at the end.
@@ -237,6 +280,16 @@ void AppendUserKeyWithMaxTimestamp(std::string* result, const Slice& key,
 void PadInternalKeyWithMinTimestamp(std::string* result, const Slice& key,
                                     size_t ts_sz);
 
+// `key` is an internal key containing a user key without timestamp. Create a
+// new key in *result by padding a max timestamp of size `ts_sz` to the user key
+// and copying the remaining internal key bytes.
+//
+// input [internal key]:   <user_provided_key | seqno + type>
+// output before: empty
+// output after:           <user_provided_key | max_ts | seqno + type>
+void PadInternalKeyWithMaxTimestamp(std::string* result, const Slice& key,
+                                    size_t ts_sz);
+
 // `key` is an internal key containing a user key with timestamp of size
 // `ts_sz`. Create a new internal key in *result by stripping the timestamp from
 // the user key and copying the remaining internal key bytes.
@@ -355,6 +408,7 @@ class InternalKeyComparator
 
   // Same as Compare except that it excludes the value type from comparison
   int CompareKeySeq(const Slice& a, const Slice& b) const;
+  int CompareKeySeq(const ParsedInternalKey& a, const Slice& b) const;
 
   const Comparator* user_comparator() const {
     return user_comparator_.user_comparator();
@@ -422,7 +476,7 @@ class InternalKey {
 
   void Set(const Slice& _user_key_with_ts, SequenceNumber s, ValueType t,
            const Slice& ts) {
-    ParsedInternalKey pik = ParsedInternalKey(_user_key_with_ts, s, t);
+    ParsedInternalKey pik(_user_key_with_ts, s, t);
     // Should not call pik.SetTimestamp() directly as it overwrites the buffer
     // containing _user_key.
     SetFrom(pik, ts);
@@ -450,7 +504,7 @@ class InternalKey {
     AppendInternalKeyFooter(&rep_, s, t);
   }
 
-  std::string DebugString(bool hex) const;
+  std::string DebugString(bool hex, const Comparator* ucmp = nullptr) const;
 };
 
 inline int InternalKeyComparator::Compare(const InternalKey& a,
@@ -820,19 +874,19 @@ class InternalKeySliceTransform : public SliceTransform {
   explicit InternalKeySliceTransform(const SliceTransform* transform)
       : transform_(transform) {}
 
-  virtual const char* Name() const override { return transform_->Name(); }
+  const char* Name() const override { return transform_->Name(); }
 
-  virtual Slice Transform(const Slice& src) const override {
+  Slice Transform(const Slice& src) const override {
     auto user_key = ExtractUserKey(src);
     return transform_->Transform(user_key);
   }
 
-  virtual bool InDomain(const Slice& src) const override {
+  bool InDomain(const Slice& src) const override {
     auto user_key = ExtractUserKey(src);
     return transform_->InDomain(user_key);
   }
 
-  virtual bool InRange(const Slice& dst) const override {
+  bool InRange(const Slice& dst) const override {
     auto user_key = ExtractUserKey(dst);
     return transform_->InRange(user_key);
   }
@@ -859,7 +913,8 @@ bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key, bool cf_record);
 // resulting from this call will include timestamp.
 Status ReadRecordFromWriteBatch(Slice* input, char* tag,
                                 uint32_t* column_family, Slice* key,
-                                Slice* value, Slice* blob, Slice* xid);
+                                Slice* value, Slice* blob, Slice* xid,
+                                uint64_t* write_unix_time);
 
 // When user call DeleteRange() to delete a range of keys,
 // we will store a serialized RangeTombstone in MemTable and SST.
@@ -882,17 +937,25 @@ struct RangeTombstone {
   // User-defined timestamp is enabled, `sk` and `ek` should be user key
   // with timestamp, `ts` will replace the timestamps in `sk` and
   // `ek`.
-  RangeTombstone(Slice sk, Slice ek, SequenceNumber sn, Slice ts)
-      : seq_(sn), ts_(ts) {
-    assert(!ts.empty());
+  // When `logical_strip_timestamp` is true, the timestamps in `sk` and `ek`
+  // will be replaced with min timestamp.
+  RangeTombstone(Slice sk, Slice ek, SequenceNumber sn, Slice ts,
+                 bool logical_strip_timestamp)
+      : seq_(sn) {
+    const size_t ts_sz = ts.size();
+    assert(ts_sz > 0);
     pinned_start_key_.reserve(sk.size());
-    pinned_start_key_.append(sk.data(), sk.size() - ts.size());
-    pinned_start_key_.append(ts.data(), ts.size());
     pinned_end_key_.reserve(ek.size());
-    pinned_end_key_.append(ek.data(), ek.size() - ts.size());
-    pinned_end_key_.append(ts.data(), ts.size());
+    if (logical_strip_timestamp) {
+      AppendUserKeyWithMinTimestamp(&pinned_start_key_, sk, ts_sz);
+      AppendUserKeyWithMinTimestamp(&pinned_end_key_, ek, ts_sz);
+    } else {
+      AppendUserKeyWithDifferentTimestamp(&pinned_start_key_, sk, ts);
+      AppendUserKeyWithDifferentTimestamp(&pinned_end_key_, ek, ts);
+    }
     start_key_ = pinned_start_key_;
     end_key_ = pinned_end_key_;
+    ts_ = Slice(pinned_start_key_.data() + sk.size() - ts_sz, ts_sz);
   }
 
   RangeTombstone(ParsedInternalKey parsed_key, Slice value) {
@@ -976,6 +1039,26 @@ inline int InternalKeyComparator::CompareKeySeq(const Slice& akey,
   return r;
 }
 
+inline int InternalKeyComparator::CompareKeySeq(const ParsedInternalKey& a,
+                                                const Slice& b) const {
+  // Order by:
+  //    increasing user key (according to user-supplied comparator)
+  //    decreasing sequence number
+  int r = user_comparator_.Compare(a.user_key, ExtractUserKey(b));
+  if (r == 0) {
+    // Shift the number to exclude the last byte which contains the value type
+    const uint64_t anum = a.sequence;
+    const uint64_t bnum =
+        DecodeFixed64(b.data() + b.size() - kNumInternalBytes) >> 8;
+    if (anum > bnum) {
+      r = -1;
+    } else if (anum < bnum) {
+      r = +1;
+    }
+  }
+  return r;
+}
+
 inline int InternalKeyComparator::Compare(const Slice& a,
                                           SequenceNumber a_global_seqno,
                                           const Slice& b,

package/deps/rocksdb/rocksdb/db/deletefile_test.cc

@@ -7,9 +7,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-
-#include <stdlib.h>
-
+#include <cstdlib>
 #include <map>
 #include <string>
 #include <vector>
@@ -106,7 +104,7 @@ class DeleteFileTest : public DBTestBase {
     ASSERT_OK(env_->GetChildren(dir, &filenames));
 
     int log_cnt = 0, sst_cnt = 0, manifest_cnt = 0;
-    for (auto file : filenames) {
+    for (const auto& file : filenames) {
       uint64_t number;
       FileType type;
       if (ParseFileName(file, &number, &type)) {
@@ -127,7 +125,7 @@ class DeleteFileTest : public DBTestBase {
   }
 
   static void DoSleep(void* arg) {
-    auto test = reinterpret_cast<DeleteFileTest*>(arg);
+    auto test = static_cast<DeleteFileTest*>(arg);
     test->env_->SleepForMicroseconds(2 * 1000 * 1000);
   }
 
@@ -148,9 +146,9 @@ TEST_F(DeleteFileTest, AddKeysAndQueryLevels) {
   std::vector<LiveFileMetaData> metadata;
   db_->GetLiveFilesMetaData(&metadata);
 
-  std::string level1file = "";
+  std::string level1file;
   int level1keycount = 0;
-  std::string level2file = "";
+  std::string level2file;
   int level2keycount = 0;
   int level1index = 0;
   int level2index = 1;
@@ -225,6 +223,49 @@ TEST_F(DeleteFileTest, PurgeObsoleteFilesTest) {
   CheckFileTypeCounts(dbname_, 0, 1, 1);
 }
 
+TEST_F(DeleteFileTest, WaitForCompactWithWaitForPurgeOptionTest) {
+  Options options = CurrentOptions();
+  SetOptions(&options);
+  Destroy(options);
+  options.create_if_missing = true;
+  Reopen(options);
+
+  std::string first("0"), last("999999");
+  CompactRangeOptions compact_options;
+  compact_options.change_level = true;
+  compact_options.target_level = 2;
+  Slice first_slice(first), last_slice(last);
+
+  CreateTwoLevels();
+  Iterator* itr = nullptr;
+  ReadOptions read_options;
+  read_options.background_purge_on_iterator_cleanup = true;
+  itr = db_->NewIterator(read_options);
+  ASSERT_OK(itr->status());
+  ASSERT_OK(db_->CompactRange(compact_options, &first_slice, &last_slice));
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::BGWorkPurge:start", "DeleteFileTest::WaitForPurgeTest"},
+       {"DBImpl::WaitForCompact:InsideLoop",
+        "DBImpl::BackgroundCallPurge:beforeMutexLock"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  delete itr;
+
+  TEST_SYNC_POINT("DeleteFileTest::WaitForPurgeTest");
+  // At this point, purge got started, but can't finish due to sync points
+  // not purged yet
+  CheckFileTypeCounts(dbname_, 0, 3, 1);
+
+  // The sync point in WaitForCompact should unblock the purge
+  WaitForCompactOptions wait_for_compact_options;
+  wait_for_compact_options.wait_for_purge = true;
+  Status s = dbfull()->WaitForCompact(wait_for_compact_options);
+  ASSERT_OK(s);
+
+  // Now files should be purged
+  CheckFileTypeCounts(dbname_, 0, 1, 1);
+}
+
 TEST_F(DeleteFileTest, BackgroundPurgeIteratorTest) {
   Options options = CurrentOptions();
   SetOptions(&options);
@@ -602,4 +643,3 @@ int main(int argc, char** argv) {
   RegisterCustomObjects(argc, argv);
   return RUN_ALL_TESTS();
 }
-