@nxtedition/rocksdb 8.0.1 → 8.0.2

package/deps/rocksdb/rocksdb/table/merging_iterator.h

@@ -12,6 +12,7 @@
 #include "db/range_del_aggregator.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/types.h"
+#include "table/iterator_wrapper.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -45,7 +46,8 @@ class MergeIteratorBuilder {
   // comparator: the comparator used in merging comparator
   // arena: where the merging iterator needs to be allocated from.
   explicit MergeIteratorBuilder(const InternalKeyComparator* comparator,
-                                Arena* arena, bool prefix_seek_mode = false);
+                                Arena* arena, bool prefix_seek_mode = false,
+                                const Slice* iterate_upper_bound = nullptr);
   ~MergeIteratorBuilder();
 
   // Add iter to the merging iterator.
@@ -88,4 +90,83 @@ class MergeIteratorBuilder {
       range_del_iter_ptrs_;
 };
 
+// For merging iterator to process range tombstones, we treat the start and end
+// keys of a range tombstone as point keys and put them into the minHeap/maxHeap
+// used in merging iterator. Take minHeap for example, we are able to keep track
+// of currently "active" range tombstones (the ones whose start keys are popped
+// but end keys are still in the heap) in `active_`. This `active_` set of range
+// tombstones is then used to quickly determine whether the point key at heap
+// top is deleted (by heap property, the point key at heap top must be within
+// internal key range of active range tombstones).
+//
+// The HeapItem struct represents 3 types of elements in the minHeap/maxHeap:
+// point key and the start and end keys of a range tombstone.
+struct HeapItem {
+  HeapItem() = default;
+
+  enum Type { ITERATOR, DELETE_RANGE_START, DELETE_RANGE_END };
+  IteratorWrapper iter;
+  size_t level = 0;
+  ParsedInternalKey parsed_ikey;
+  std::string range_tombstone_key;
+  // Will be overwritten before use, initialize here so compiler does not
+  // complain.
+  Type type = ITERATOR;
+
+  explicit HeapItem(size_t _level, InternalIteratorBase<Slice>* _iter)
+      : level(_level), type(Type::ITERATOR) {
+    iter.Set(_iter);
+  }
+
+  void SetTombstoneKey(ParsedInternalKey&& pik) {
+    // op_type is already initialized in MergingIterator::Finish().
+    parsed_ikey.user_key = pik.user_key;
+    parsed_ikey.sequence = pik.sequence;
+  }
+
+  void SetTombstoneForCompaction(const ParsedInternalKey&& pik) {
+    range_tombstone_key.clear();
+    AppendInternalKey(&range_tombstone_key, pik);
+  }
+
+  Slice key() const {
+    if (LIKELY(type == ITERATOR)) {
+      return iter.key();
+    }
+    return range_tombstone_key;
+  }
+
+  bool IsDeleteRangeSentinelKey() const {
+    if (LIKELY(type == ITERATOR)) {
+      return iter.IsDeleteRangeSentinelKey();
+    }
+    return false;
+  }
+};
+
+class MinHeapItemComparator {
+ public:
+  explicit MinHeapItemComparator(const InternalKeyComparator* comparator)
+      : comparator_(comparator) {}
+  bool operator()(HeapItem* a, HeapItem* b) const {
+    if (LIKELY(a->type == HeapItem::ITERATOR)) {
+      if (LIKELY(b->type == HeapItem::ITERATOR)) {
+        return comparator_->Compare(a->iter.key(), b->iter.key()) > 0;
+      } else {
+        return comparator_->Compare(a->iter.key(), b->parsed_ikey) > 0;
+      }
+    } else {
+      if (LIKELY(b->type == HeapItem::ITERATOR)) {
+        return comparator_->Compare(a->parsed_ikey, b->iter.key()) > 0;
+      } else {
+        return comparator_->Compare(a->parsed_ikey, b->parsed_ikey) > 0;
+      }
+    }
+  }
+
+ private:
+  const InternalKeyComparator* comparator_;
+};
+
+using MergerMinIterHeap = BinaryHeap<HeapItem*, MinHeapItemComparator>;
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/table/meta_blocks.cc

@@ -479,8 +479,8 @@ Status ReadMetaIndexBlockInFile(RandomAccessFileReader* file,
                                 Footer* footer_out) {
   Footer footer;
   IOOptions opts;
-  auto s = ReadFooterFromFile(opts, file, prefetch_buffer, file_size, &footer,
-                              table_magic_number);
+  auto s = ReadFooterFromFile(opts, file, *ioptions.fs, prefetch_buffer,
+                              file_size, &footer, table_magic_number);
   if (!s.ok()) {
     return s;
   }

package/deps/rocksdb/rocksdb/table/sst_file_dumper.cc

@@ -113,7 +113,7 @@ Status SstFileDumper::GetTableReader(const std::string& file_path) {
                                  static_cast<size_t>(prefetch_size),
                                  Env::IO_TOTAL /* rate_limiter_priority */);
 
-    s = ReadFooterFromFile(opts, file_.get(), &prefetch_buffer, file_size,
+    s = ReadFooterFromFile(opts, file_.get(), *fs, &prefetch_buffer, file_size,
                            &footer);
   }
   if (s.ok()) {

package/deps/rocksdb/rocksdb/table/table_test.cc

@@ -2256,8 +2256,9 @@ TEST_P(BlockBasedTableTest, BadChecksumType) {
   const MutableCFOptions new_moptions(options);
   Status s = c.Reopen(new_ioptions, new_moptions);
   ASSERT_NOK(s);
+  // "test" is file name
   ASSERT_EQ(s.ToString(),
-            "Corruption: Corrupt or unsupported checksum type: 123");
+            "Corruption: Corrupt or unsupported checksum type: 123 in test");
 }
 
 namespace {
@@ -4806,9 +4807,9 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
 
     Footer footer;
     IOOptions opts;
-    ASSERT_OK(ReadFooterFromFile(opts, file, nullptr /* prefetch_buffer */,
-                                 file_size, &footer,
-                                 kBlockBasedTableMagicNumber));
+    ASSERT_OK(ReadFooterFromFile(opts, file, *FileSystem::Default(),
+                                 nullptr /* prefetch_buffer */, file_size,
+                                 &footer, kBlockBasedTableMagicNumber));
 
     auto BlockFetchHelper = [&](const BlockHandle& handle, BlockType block_type,
                                 BlockContents* contents) {
@@ -4892,7 +4893,7 @@ TEST_P(BlockBasedTableTest, PropertiesMetaBlockLast) {
   // read footer
   Footer footer;
   IOOptions opts;
-  ASSERT_OK(ReadFooterFromFile(opts, table_reader.get(),
+  ASSERT_OK(ReadFooterFromFile(opts, table_reader.get(), *FileSystem::Default(),
                                nullptr /* prefetch_buffer */, table_size,
                                &footer, kBlockBasedTableMagicNumber));
 
@@ -4970,7 +4971,7 @@ TEST_P(BlockBasedTableTest, SeekMetaBlocks) {
   // read footer
   Footer footer;
   IOOptions opts;
-  ASSERT_OK(ReadFooterFromFile(opts, table_reader.get(),
+  ASSERT_OK(ReadFooterFromFile(opts, table_reader.get(), *FileSystem::Default(),
                                nullptr /* prefetch_buffer */, table_size,
                                &footer, kBlockBasedTableMagicNumber));
 

package/deps/rocksdb/rocksdb/test_util/testutil.h

@@ -363,6 +363,16 @@ class SleepingBackgroundTask {
         done_with_sleep_(false),
         sleeping_(false) {}
 
+  ~SleepingBackgroundTask() {
+    MutexLock l(&mutex_);
+    should_sleep_ = false;
+    while (sleeping_) {
+      assert(!should_sleep_);
+      bg_cv_.SignalAll();
+      bg_cv_.Wait();
+    }
+  }
+
   bool IsSleeping() {
     MutexLock l(&mutex_);
     return sleeping_;

package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc

@@ -37,7 +37,6 @@
 #include <thread>
 #include <unordered_map>
 
-#include "cache/fast_lru_cache.h"
 #include "db/db_impl/db_impl.h"
 #include "db/malloc_stats.h"
 #include "db/version_set.h"
@@ -3055,11 +3054,6 @@ class Benchmark {
                                     FLAGS_block_size /*estimated_entry_charge*/,
                                     FLAGS_cache_numshardbits)
           .MakeSharedCache();
-    } else if (FLAGS_cache_type == "fast_lru_cache") {
-      return NewFastLRUCache(static_cast<size_t>(capacity), FLAGS_block_size,
-                             FLAGS_cache_numshardbits,
-                             false /*strict_capacity_limit*/,
-                             kDefaultCacheMetadataChargePolicy);
     } else if (FLAGS_cache_type == "lru_cache") {
       LRUCacheOptions opts(
           static_cast<size_t>(capacity), FLAGS_cache_numshardbits,

package/deps/rocksdb/rocksdb/trace_replay/block_cache_tracer.h

@@ -129,11 +129,11 @@ class BlockCacheTraceWriterImpl : public BlockCacheTraceWriter {
   // Pass Slice references to avoid copy.
   Status WriteBlockAccess(const BlockCacheTraceRecord& record,
                           const Slice& block_key, const Slice& cf_name,
-                          const Slice& referenced_key);
+                          const Slice& referenced_key) override;
 
   // Write a trace header at the beginning, typically on initiating a trace,
   // with some metadata like a magic number and RocksDB version.
-  Status WriteHeader();
+  Status WriteHeader() override;
 
  private:
   SystemClock* clock_;

package/deps/rocksdb/rocksdb/util/bloom_test.cc

@@ -806,7 +806,7 @@ TEST_P(FullBloomTest, Schema) {
 struct RawFilterTester {
   // Buffer, from which we always return a tail Slice, so the
   // last five bytes are always the metadata bytes.
-  std::array<char, 3000> data_;
+  std::array<char, 3000> data_{};
   // Points five bytes from the end
   char* metadata_ptr_;
 

package/deps/rocksdb/rocksdb/util/status.cc

@@ -69,6 +69,13 @@ Status::Status(Code _code, SubCode _subcode, const Slice& msg,
   state_.reset(result);
 }
 
+Status Status::CopyAppendMessage(const Status& s, const Slice& delim,
+                                 const Slice& msg) {
+  // (No attempt at efficiency)
+  return Status(s.code(), s.subcode(), s.severity(),
+                std::string(s.getState()) + delim.ToString() + msg.ToString());
+}
+
 std::string Status::ToString() const {
 #ifdef ROCKSDB_ASSERT_STATUS_CHECKED
   checked_ = true;

package/deps/rocksdb/rocksdb/utilities/backup/backup_engine.cc

@@ -1635,6 +1635,11 @@ IOStatus BackupEngineImpl::DeleteBackupNoGC(BackupID backup_id) {
       return io_s;
     }
     backups_.erase(backup);
+    if (backups_.empty()) {
+      latest_valid_backup_id_ = 0;
+    } else {
+      latest_valid_backup_id_ = backups_.rbegin()->first;
+    }
   } else {
     auto corrupt = corrupt_backups_.find(backup_id);
     if (corrupt == corrupt_backups_.end()) {

package/deps/rocksdb/rocksdb/utilities/backup/backup_engine_test.cc

@@ -1213,6 +1213,10 @@ TEST_P(BackupEngineTestWithParam, OnlineIntegrationTest) {
   // check backup 5
   AssertBackupConsistency(5, 0, max_key);
 
+  // check that "latest backup" still works after deleting latest
+  ASSERT_OK(backup_engine_->DeleteBackup(5));
+  AssertBackupConsistency(0, 0, 3 * keys_iteration, max_key);
+
   CloseBackupEngine();
 }
 #endif  // !defined(ROCKSDB_VALGRIND_RUN) || defined(ROCKSDB_FULL_VALGRIND_RUN)

package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.cc

@@ -280,7 +280,7 @@ IOStatus CacheDumpedLoaderImpl::RestoreCacheEntriesToSecondaryCache() {
 
   // Step 3: read out the rest of the blocks from the reader. The loop will stop
   // either I/O status is not ok or we reach to the the end.
-  while (io_s.ok() && dump_unit.type != CacheDumpUnitType::kFooter) {
+  while (io_s.ok()) {
     dump_unit.reset();
     data.clear();
     // read the content and store in the dump_unit
@@ -288,74 +288,14 @@ IOStatus CacheDumpedLoaderImpl::RestoreCacheEntriesToSecondaryCache() {
     if (!io_s.ok()) {
       break;
     }
+    if (dump_unit.type == CacheDumpUnitType::kFooter) {
+      break;
+    }
     // Create the uncompressed_block based on the information in the dump_unit
     // (There is no block trailer here compatible with block-based SST file.)
-    BlockContents uncompressed_block(
-        Slice(static_cast<char*>(dump_unit.value), dump_unit.value_len));
-    Cache::CacheItemHelper* helper = nullptr;
-    Statistics* statistics = nullptr;
-    Status s = Status::OK();
-    // according to the block type, get the helper callback function and create
-    // the corresponding block
-    switch (dump_unit.type) {
-      case CacheDumpUnitType::kFilter: {
-        helper = BlocklikeTraits<ParsedFullFilterBlock>::GetCacheItemHelper(
-            BlockType::kFilter);
-        std::unique_ptr<ParsedFullFilterBlock> block_holder;
-        block_holder.reset(BlocklikeTraits<ParsedFullFilterBlock>::Create(
-            std::move(uncompressed_block), toptions_.read_amp_bytes_per_bit,
-            statistics, false, toptions_.filter_policy.get()));
-        if (helper != nullptr) {
-          s = secondary_cache_->Insert(dump_unit.key,
-                                       (void*)(block_holder.get()), helper);
-        }
-        break;
-      }
-      case CacheDumpUnitType::kData: {
-        helper = BlocklikeTraits<Block>::GetCacheItemHelper(BlockType::kData);
-        std::unique_ptr<Block> block_holder;
-        block_holder.reset(BlocklikeTraits<Block>::Create(
-            std::move(uncompressed_block), toptions_.read_amp_bytes_per_bit,
-            statistics, false, toptions_.filter_policy.get()));
-        if (helper != nullptr) {
-          s = secondary_cache_->Insert(dump_unit.key,
-                                       (void*)(block_holder.get()), helper);
-        }
-        break;
-      }
-      case CacheDumpUnitType::kIndex: {
-        helper = BlocklikeTraits<Block>::GetCacheItemHelper(BlockType::kIndex);
-        std::unique_ptr<Block> block_holder;
-        block_holder.reset(BlocklikeTraits<Block>::Create(
-            std::move(uncompressed_block), 0, statistics, false,
-            toptions_.filter_policy.get()));
-        if (helper != nullptr) {
-          s = secondary_cache_->Insert(dump_unit.key,
-                                       (void*)(block_holder.get()), helper);
-        }
-        break;
-      }
-      case CacheDumpUnitType::kFilterMetaBlock: {
-        helper = BlocklikeTraits<Block>::GetCacheItemHelper(
-            BlockType::kFilterPartitionIndex);
-        std::unique_ptr<Block> block_holder;
-        block_holder.reset(BlocklikeTraits<Block>::Create(
-            std::move(uncompressed_block), toptions_.read_amp_bytes_per_bit,
-            statistics, false, toptions_.filter_policy.get()));
-        if (helper != nullptr) {
-          s = secondary_cache_->Insert(dump_unit.key,
-                                       (void*)(block_holder.get()), helper);
-        }
-        break;
-      }
-      case CacheDumpUnitType::kFooter:
-        break;
-      case CacheDumpUnitType::kDeprecatedFilterBlock:
-        // Obsolete
-        break;
-      default:
-        continue;
-    }
+    Slice content =
+        Slice(static_cast<char*>(dump_unit.value), dump_unit.value_len);
+    Status s = secondary_cache_->InsertSaved(dump_unit.key, content);
     if (!s.ok()) {
       io_s = status_to_io_status(std::move(s));
     }

package/deps/rocksdb/rocksdb/utilities/cache_dump_load_impl.h

@@ -128,11 +128,10 @@ class CacheDumperImpl : public CacheDumper {
 class CacheDumpedLoaderImpl : public CacheDumpedLoader {
  public:
   CacheDumpedLoaderImpl(const CacheDumpOptions& dump_options,
-                        const BlockBasedTableOptions& toptions,
+                        const BlockBasedTableOptions& /*toptions*/,
                         const std::shared_ptr<SecondaryCache>& secondary_cache,
                         std::unique_ptr<CacheDumpReader>&& reader)
       : options_(dump_options),
-        toptions_(toptions),
         secondary_cache_(secondary_cache),
         reader_(std::move(reader)) {}
   ~CacheDumpedLoaderImpl() {}
@@ -145,7 +144,6 @@ class CacheDumpedLoaderImpl : public CacheDumpedLoader {
   IOStatus ReadCacheBlock(std::string* data, DumpUnit* dump_unit);
 
   CacheDumpOptions options_;
-  const BlockBasedTableOptions& toptions_;
   std::shared_ptr<SecondaryCache> secondary_cache_;
   std::unique_ptr<CacheDumpReader> reader_;
   UnorderedMap<Cache::DeleterFn, CacheEntryRole> role_map_;

package/deps/rocksdb/rocksdb/utilities/checkpoint/checkpoint_impl.cc

@@ -382,6 +382,7 @@ Status CheckpointImpl::ExportColumnFamily(
         live_file_metadata.largestkey = std::move(file_metadata.largestkey);
         live_file_metadata.oldest_blob_file_number =
             file_metadata.oldest_blob_file_number;
+        live_file_metadata.epoch_number = file_metadata.epoch_number;
         live_file_metadata.level = level_metadata.level;
         result_metadata->files.push_back(live_file_metadata);
       }

package/deps/rocksdb/rocksdb/utilities/transactions/transaction_test.cc

@@ -6530,6 +6530,65 @@ TEST_P(TransactionTest, WriteWithBulkCreatedColumnFamilies) {
   cf_handles.clear();
 }
 
+TEST_P(TransactionTest, LockWal) {
+  const TxnDBWritePolicy write_policy = std::get<2>(GetParam());
+  if (TxnDBWritePolicy::WRITE_COMMITTED != write_policy) {
+    ROCKSDB_GTEST_BYPASS("Test only write-committed for now");
+    return;
+  }
+  ASSERT_OK(ReOpen());
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"TransactionTest::LockWal:AfterLockWal",
+        "TransactionTest::LockWal:BeforePrepareTxn2"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  std::unique_ptr<Transaction> txn0;
+  WriteOptions wopts;
+  wopts.no_slowdown = true;
+  txn0.reset(db->BeginTransaction(wopts, TransactionOptions()));
+  ASSERT_OK(txn0->SetName("txn0"));
+  ASSERT_OK(txn0->Put("foo", "v0"));
+
+  std::unique_ptr<Transaction> txn1;
+  txn1.reset(db->BeginTransaction(wopts, TransactionOptions()));
+  ASSERT_OK(txn1->SetName("txn1"));
+  ASSERT_OK(txn1->Put("dummy", "v0"));
+  ASSERT_OK(txn1->Prepare());
+
+  std::unique_ptr<Transaction> txn2;
+  port::Thread worker([&]() {
+    txn2.reset(db->BeginTransaction(WriteOptions(), TransactionOptions()));
+    ASSERT_OK(txn2->SetName("txn2"));
+    ASSERT_OK(txn2->Put("bar", "v0"));
+    TEST_SYNC_POINT("TransactionTest::LockWal:BeforePrepareTxn2");
+    ASSERT_OK(txn2->Prepare());
+    ASSERT_OK(txn2->Commit());
+  });
+  ASSERT_OK(db->LockWAL());
+  // txn0 cannot prepare
+  Status s = txn0->Prepare();
+  ASSERT_TRUE(s.IsIncomplete());
+  // txn1 cannot commit
+  s = txn1->Commit();
+  ASSERT_TRUE(s.IsIncomplete());
+
+  TEST_SYNC_POINT("TransactionTest::LockWal:AfterLockWal");
+
+  ASSERT_OK(db->UnlockWAL());
+  txn0.reset();
+
+  txn0.reset(db->BeginTransaction(wopts, TransactionOptions()));
+  ASSERT_OK(txn0->SetName("txn0_1"));
+  ASSERT_OK(txn0->Put("foo", "v1"));
+  ASSERT_OK(txn0->Prepare());
+  ASSERT_OK(txn0->Commit());
+  worker.join();
+
+  SyncPoint::GetInstance()->DisableProcessing();
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb.gyp

@@ -168,8 +168,8 @@
         "rocksdb/cache/charged_cache.cc",
         "rocksdb/cache/clock_cache.cc",
         "rocksdb/cache/compressed_secondary_cache.cc",
-        "rocksdb/cache/fast_lru_cache.cc",
         "rocksdb/cache/lru_cache.cc",
+				"rocksdb/cache/secondary_cache.cc",
         "rocksdb/cache/sharded_cache.cc",
         "rocksdb/db/arena_wrapped_db_iter.cc",
         "rocksdb/db/blob/blob_contents.cc",
@@ -339,6 +339,7 @@
         "rocksdb/table/block_based/reader_common.cc",
         "rocksdb/table/block_based/uncompression_dict_reader.cc",
         "rocksdb/table/block_fetcher.cc",
+				"rocksdb/table/compaction_merging_iterator.cc",
         "rocksdb/table/cuckoo/cuckoo_table_builder.cc",
         "rocksdb/table/cuckoo/cuckoo_table_factory.cc",
         "rocksdb/table/cuckoo/cuckoo_table_reader.cc",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "8.0.1",
+  "version": "8.0.2",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",