@nxtedition/rocksdb 8.2.0 → 8.2.2

package/deps/rocksdb/rocksdb/db/version_edit.cc

@@ -238,6 +238,12 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
                   f.compensated_range_deletion_size);
       PutLengthPrefixedSlice(dst, Slice(compensated_range_deletion_size));
     }
+    if (f.tail_size) {
+      PutVarint32(dst, NewFileCustomTag::kTailSize);
+      std::string varint_tail_size;
+      PutVarint64(&varint_tail_size, f.tail_size);
+      PutLengthPrefixedSlice(dst, Slice(varint_tail_size));
+    }
 
     TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:NewFile4:CustomizeFields",
                              dst);
@@ -416,6 +422,11 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
             return "Invalid compensated range deletion size";
           }
           break;
+        case kTailSize:
+          if (!GetVarint64(&field, &f.tail_size)) {
+            return "invalid tail start offset";
+          }
+          break;
         default:
           if ((custom_tag & kCustomTagNonSafeIgnoreMask) != 0) {
             // Should not proceed if cannot understand it
@@ -851,6 +862,8 @@ std::string VersionEdit::DebugString(bool hex_key) const {
       InternalUniqueIdToExternal(&id);
       r.append(UniqueIdToHumanString(EncodeUniqueIdBytes(&id)));
     }
+    r.append(" tail size:");
+    AppendNumberTo(&r, f.tail_size);
   }
 
   for (const auto& blob_file_addition : blob_file_additions_) {
@@ -966,6 +979,7 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
         // permanent
         jw << "Temperature" << static_cast<int>(f.temperature);
       }
+      jw << "TailSize" << f.tail_size;
       jw.EndArrayedObject();
     }
 

package/deps/rocksdb/rocksdb/db/version_edit.h

@@ -90,6 +90,7 @@ enum NewFileCustomTag : uint32_t {
   kUniqueId = 12,
   kEpochNumber = 13,
   kCompensatedRangeDeletionSize = 14,
+  kTailSize = 15,
 
   // If this bit for the custom tag is set, opening DB should fail if
   // we don't know this field.
@@ -238,6 +239,10 @@ struct FileMetaData {
   // SST unique id
   UniqueId64x2 unique_id{};
 
+  // Size of the "tail" part of a SST file
+  // "Tail" refers to all blocks after data blocks till the end of the SST file
+  uint64_t tail_size = 0;
+
   FileMetaData() = default;
 
   FileMetaData(uint64_t file, uint32_t file_path_id, uint64_t file_size,
@@ -249,7 +254,8 @@ struct FileMetaData {
                uint64_t _epoch_number, const std::string& _file_checksum,
                const std::string& _file_checksum_func_name,
                UniqueId64x2 _unique_id,
-               const uint64_t _compensated_range_deletion_size)
+               const uint64_t _compensated_range_deletion_size,
+               uint64_t _tail_size)
       : fd(file, file_path_id, file_size, smallest_seq, largest_seq),
         smallest(smallest_key),
         largest(largest_key),
@@ -262,7 +268,8 @@ struct FileMetaData {
         epoch_number(_epoch_number),
         file_checksum(_file_checksum),
         file_checksum_func_name(_file_checksum_func_name),
-        unique_id(std::move(_unique_id)) {
+        unique_id(std::move(_unique_id)),
+        tail_size(_tail_size) {
     TEST_SYNC_POINT_CALLBACK("FileMetaData::FileMetaData", this);
   }
 
@@ -446,7 +453,8 @@ class VersionEdit {
                uint64_t epoch_number, const std::string& file_checksum,
                const std::string& file_checksum_func_name,
                const UniqueId64x2& unique_id,
-               const uint64_t compensated_range_deletion_size) {
+               const uint64_t compensated_range_deletion_size,
+               uint64_t tail_size) {
     assert(smallest_seqno <= largest_seqno);
     new_files_.emplace_back(
         level,
@@ -455,7 +463,7 @@ class VersionEdit {
                      temperature, oldest_blob_file_number, oldest_ancester_time,
                      file_creation_time, epoch_number, file_checksum,
                      file_checksum_func_name, unique_id,
-                     compensated_range_deletion_size));
+                     compensated_range_deletion_size, tail_size));
     if (!HasLastSequence() || largest_seqno > GetLastSequence()) {
       SetLastSequence(largest_seqno);
     }

package/deps/rocksdb/rocksdb/db/version_edit_handler.cc

@@ -155,8 +155,9 @@ VersionEditHandler::VersionEditHandler(
     bool read_only, std::vector<ColumnFamilyDescriptor> column_families,
     VersionSet* version_set, bool track_missing_files,
     bool no_error_if_files_missing, const std::shared_ptr<IOTracer>& io_tracer,
-    bool skip_load_table_files, EpochNumberRequirement epoch_number_requirement)
-    : VersionEditHandlerBase(),
+    const ReadOptions& read_options, bool skip_load_table_files,
+    EpochNumberRequirement epoch_number_requirement)
+    : VersionEditHandlerBase(read_options),
       read_only_(read_only),
       column_families_(std::move(column_families)),
       version_set_(version_set),
@@ -480,7 +481,8 @@ void VersionEditHandler::CheckIterationResult(const log::Reader& reader,
 
 ColumnFamilyData* VersionEditHandler::CreateCfAndInit(
     const ColumnFamilyOptions& cf_options, const VersionEdit& edit) {
-  ColumnFamilyData* cfd = version_set_->CreateColumnFamily(cf_options, &edit);
+  ColumnFamilyData* cfd =
+      version_set_->CreateColumnFamily(cf_options, read_options_, &edit);
   assert(cfd != nullptr);
   cfd->set_initialized();
   assert(builders_.find(edit.column_family_) == builders_.end());
@@ -537,7 +539,7 @@ Status VersionEditHandler::MaybeCreateVersion(const VersionEdit& /*edit*/,
     if (s.ok()) {
       // Install new version
       v->PrepareAppend(
-          *cfd->GetLatestMutableCFOptions(),
+          *cfd->GetLatestMutableCFOptions(), read_options_,
           !(version_set_->db_options_->skip_stats_update_on_db_open));
       version_set_->AppendVersion(cfd, v);
     } else {
@@ -564,12 +566,13 @@ Status VersionEditHandler::LoadTables(ColumnFamilyData* cfd,
   assert(builder_iter->second != nullptr);
   VersionBuilder* builder = builder_iter->second->version_builder();
   assert(builder);
+  const MutableCFOptions* moptions = cfd->GetLatestMutableCFOptions();
   Status s = builder->LoadTableHandlers(
       cfd->internal_stats(),
       version_set_->db_options_->max_file_opening_threads,
       prefetch_index_and_filter_in_cache, is_initial_load,
-      cfd->GetLatestMutableCFOptions()->prefix_extractor,
-      MaxFileSizeForL0MetaPin(*cfd->GetLatestMutableCFOptions()));
+      moptions->prefix_extractor, MaxFileSizeForL0MetaPin(*moptions),
+      read_options_, moptions->block_protection_bytes_per_key);
   if ((s.IsPathNotFound() || s.IsCorruption()) && no_error_if_files_missing_) {
     s = Status::OK();
   }
@@ -647,11 +650,12 @@ Status VersionEditHandler::ExtractInfoFromVersionEdit(ColumnFamilyData* cfd,
 VersionEditHandlerPointInTime::VersionEditHandlerPointInTime(
     bool read_only, std::vector<ColumnFamilyDescriptor> column_families,
     VersionSet* version_set, const std::shared_ptr<IOTracer>& io_tracer,
+    const ReadOptions& read_options,
     EpochNumberRequirement epoch_number_requirement)
     : VersionEditHandler(read_only, column_families, version_set,
                          /*track_missing_files=*/true,
                          /*no_error_if_files_missing=*/true, io_tracer,
-                         epoch_number_requirement) {}
+                         read_options, epoch_number_requirement) {}
 
 VersionEditHandlerPointInTime::~VersionEditHandlerPointInTime() {
   for (const auto& elem : versions_) {
@@ -808,15 +812,16 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersion(
       assert(builder);
     }
 
+    const MutableCFOptions* cf_opts_ptr = cfd->GetLatestMutableCFOptions();
     auto* version = new Version(cfd, version_set_, version_set_->file_options_,
-                                *cfd->GetLatestMutableCFOptions(), io_tracer_,
+                                *cf_opts_ptr, io_tracer_,
                                 version_set_->current_version_number_++,
                                 epoch_number_requirement_);
     s = builder->LoadTableHandlers(
         cfd->internal_stats(),
         version_set_->db_options_->max_file_opening_threads, false, true,
-        cfd->GetLatestMutableCFOptions()->prefix_extractor,
-        MaxFileSizeForL0MetaPin(*cfd->GetLatestMutableCFOptions()));
+        cf_opts_ptr->prefix_extractor, MaxFileSizeForL0MetaPin(*cf_opts_ptr),
+        read_options_, cf_opts_ptr->block_protection_bytes_per_key);
     if (!s.ok()) {
       delete version;
       if (s.IsCorruption()) {
@@ -827,7 +832,7 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersion(
     s = builder->SaveTo(version->storage_info());
     if (s.ok()) {
       version->PrepareAppend(
-          *cfd->GetLatestMutableCFOptions(),
+          *cfd->GetLatestMutableCFOptions(), read_options_,
           !version_set_->db_options_->skip_stats_update_on_db_open);
       auto v_iter = versions_.find(cfd->GetID());
       if (v_iter != versions_.end()) {
@@ -847,7 +852,8 @@ Status VersionEditHandlerPointInTime::VerifyFile(ColumnFamilyData* cfd,
                                                  const std::string& fpath,
                                                  int level,
                                                  const FileMetaData& fmeta) {
-  return version_set_->VerifyFileMetadata(cfd, fpath, level, fmeta);
+  return version_set_->VerifyFileMetadata(read_options_, cfd, fpath, level,
+                                          fmeta);
 }
 
 Status VersionEditHandlerPointInTime::VerifyBlobFile(
@@ -856,7 +862,9 @@ Status VersionEditHandlerPointInTime::VerifyBlobFile(
   BlobSource* blob_source = cfd->blob_source();
   assert(blob_source);
   CacheHandleGuard<BlobFileReader> blob_file_reader;
-  Status s = blob_source->GetBlobFileReader(blob_file_num, &blob_file_reader);
+
+  Status s = blob_source->GetBlobFileReader(read_options_, blob_file_num,
+                                            &blob_file_reader);
   if (!s.ok()) {
     return s;
   }

package/deps/rocksdb/rocksdb/db/version_edit_handler.h

@@ -19,8 +19,9 @@ struct FileMetaData;
 
 class VersionEditHandlerBase {
  public:
-  explicit VersionEditHandlerBase()
-      : max_manifest_read_size_(std::numeric_limits<uint64_t>::max()) {}
+  explicit VersionEditHandlerBase(const ReadOptions& read_options)
+      : read_options_(read_options),
+        max_manifest_read_size_(std::numeric_limits<uint64_t>::max()) {}
 
   virtual ~VersionEditHandlerBase() {}
 
@@ -31,8 +32,9 @@ class VersionEditHandlerBase {
   AtomicGroupReadBuffer& GetReadBuffer() { return read_buffer_; }
 
  protected:
-  explicit VersionEditHandlerBase(uint64_t max_read_size)
-      : max_manifest_read_size_(max_read_size) {}
+  explicit VersionEditHandlerBase(const ReadOptions& read_options,
+                                  uint64_t max_read_size)
+      : read_options_(read_options), max_manifest_read_size_(max_read_size) {}
   virtual Status Initialize() { return Status::OK(); }
 
   virtual Status ApplyVersionEdit(VersionEdit& edit,
@@ -45,6 +47,8 @@ class VersionEditHandlerBase {
 
   Status status_;
 
+  const ReadOptions& read_options_;
+
  private:
   AtomicGroupReadBuffer read_buffer_;
   const uint64_t max_manifest_read_size_;
@@ -52,7 +56,8 @@ class VersionEditHandlerBase {
 
 class ListColumnFamiliesHandler : public VersionEditHandlerBase {
  public:
-  ListColumnFamiliesHandler() : VersionEditHandlerBase() {}
+  explicit ListColumnFamiliesHandler(const ReadOptions& read_options)
+      : VersionEditHandlerBase(read_options) {}
 
   ~ListColumnFamiliesHandler() override {}
 
@@ -72,9 +77,9 @@ class ListColumnFamiliesHandler : public VersionEditHandlerBase {
 
 class FileChecksumRetriever : public VersionEditHandlerBase {
  public:
-  FileChecksumRetriever(uint64_t max_read_size,
+  FileChecksumRetriever(const ReadOptions& read_options, uint64_t max_read_size,
                         FileChecksumList& file_checksum_list)
-      : VersionEditHandlerBase(max_read_size),
+      : VersionEditHandlerBase(read_options, max_read_size),
         file_checksum_list_(file_checksum_list) {}
 
   ~FileChecksumRetriever() override {}
@@ -111,12 +116,13 @@ class VersionEditHandler : public VersionEditHandlerBase {
       VersionSet* version_set, bool track_missing_files,
       bool no_error_if_files_missing,
       const std::shared_ptr<IOTracer>& io_tracer,
+      const ReadOptions& read_options,
       EpochNumberRequirement epoch_number_requirement =
           EpochNumberRequirement::kMustPresent)
-      : VersionEditHandler(read_only, column_families, version_set,
-                           track_missing_files, no_error_if_files_missing,
-                           io_tracer, /*skip_load_table_files=*/false,
-                           epoch_number_requirement) {}
+      : VersionEditHandler(
+            read_only, column_families, version_set, track_missing_files,
+            no_error_if_files_missing, io_tracer, read_options,
+            /*skip_load_table_files=*/false, epoch_number_requirement) {}
 
   ~VersionEditHandler() override {}
 
@@ -137,7 +143,8 @@ class VersionEditHandler : public VersionEditHandlerBase {
       bool read_only, std::vector<ColumnFamilyDescriptor> column_families,
       VersionSet* version_set, bool track_missing_files,
       bool no_error_if_files_missing,
-      const std::shared_ptr<IOTracer>& io_tracer, bool skip_load_table_files,
+      const std::shared_ptr<IOTracer>& io_tracer,
+      const ReadOptions& read_options, bool skip_load_table_files,
       EpochNumberRequirement epoch_number_requirement =
           EpochNumberRequirement::kMustPresent);
 
@@ -212,6 +219,7 @@ class VersionEditHandlerPointInTime : public VersionEditHandler {
   VersionEditHandlerPointInTime(
       bool read_only, std::vector<ColumnFamilyDescriptor> column_families,
       VersionSet* version_set, const std::shared_ptr<IOTracer>& io_tracer,
+      const ReadOptions& read_options,
       EpochNumberRequirement epoch_number_requirement =
           EpochNumberRequirement::kMustPresent);
   ~VersionEditHandlerPointInTime() override;
@@ -238,10 +246,11 @@ class ManifestTailer : public VersionEditHandlerPointInTime {
   explicit ManifestTailer(std::vector<ColumnFamilyDescriptor> column_families,
                           VersionSet* version_set,
                           const std::shared_ptr<IOTracer>& io_tracer,
+                          const ReadOptions& read_options,
                           EpochNumberRequirement epoch_number_requirement =
                               EpochNumberRequirement::kMustPresent)
       : VersionEditHandlerPointInTime(/*read_only=*/false, column_families,
-                                      version_set, io_tracer,
+                                      version_set, io_tracer, read_options,
                                       epoch_number_requirement),
         mode_(Mode::kRecovery) {}
 
@@ -281,12 +290,13 @@ class DumpManifestHandler : public VersionEditHandler {
  public:
   DumpManifestHandler(std::vector<ColumnFamilyDescriptor> column_families,
                       VersionSet* version_set,
-                      const std::shared_ptr<IOTracer>& io_tracer, bool verbose,
-                      bool hex, bool json)
+                      const std::shared_ptr<IOTracer>& io_tracer,
+                      const ReadOptions& read_options, bool verbose, bool hex,
+                      bool json)
       : VersionEditHandler(
             /*read_only=*/true, column_families, version_set,
             /*track_missing_files=*/false,
-            /*no_error_if_files_missing=*/false, io_tracer,
+            /*no_error_if_files_missing=*/false, io_tracer, read_options,
             /*skip_load_table_files=*/true),
         verbose_(verbose),
         hex_(hex),

package/deps/rocksdb/rocksdb/db/version_edit_test.cc

@@ -45,7 +45,7 @@ TEST_F(VersionEditTest, EncodeDecode) {
                  kBig + 500 + i, kBig + 600 + i, false, Temperature::kUnknown,
                  kInvalidBlobFileNumber, 888, 678,
                  kBig + 300 + i /* epoch_number */, "234", "crc32c",
-                 kNullUniqueId64x2, 0);
+                 kNullUniqueId64x2, 0, 0);
     edit.DeleteFile(4, kBig + 700 + i);
   }
 
@@ -65,24 +65,24 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
                kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber,
                kUnknownOldestAncesterTime, kUnknownFileCreationTime,
                300 /* epoch_number */, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
+               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0);
   edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
                InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
                kBig + 601, false, Temperature::kUnknown, kInvalidBlobFileNumber,
                kUnknownOldestAncesterTime, kUnknownFileCreationTime,
                301 /* epoch_number */, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
+               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0);
   edit.AddFile(5, 302, 0, 100, InternalKey("foo", kBig + 502, kTypeValue),
                InternalKey("zoo", kBig + 602, kTypeDeletion), kBig + 502,
                kBig + 602, true, Temperature::kUnknown, kInvalidBlobFileNumber,
                666, 888, 302 /* epoch_number */, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
+               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0);
   edit.AddFile(5, 303, 0, 100, InternalKey("foo", kBig + 503, kTypeBlobIndex),
                InternalKey("zoo", kBig + 603, kTypeBlobIndex), kBig + 503,
                kBig + 603, true, Temperature::kUnknown, 1001,
                kUnknownOldestAncesterTime, kUnknownFileCreationTime,
                303 /* epoch_number */, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
+               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0);
 
   edit.DeleteFile(4, 700);
 
@@ -123,12 +123,12 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
                kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber,
                kUnknownOldestAncesterTime, kUnknownFileCreationTime,
                300 /* epoch_number */, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
+               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0);
   edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
                InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
                kBig + 601, false, Temperature::kUnknown, kInvalidBlobFileNumber,
                686, 868, 301 /* epoch_number */, "234", "crc32c",
-               kNullUniqueId64x2, 0);
+               kNullUniqueId64x2, 0, 0);
   edit.DeleteFile(4, 700);
 
   edit.SetComparatorName("foo");
@@ -177,7 +177,7 @@ TEST_F(VersionEditTest, NewFile4NotSupportedField) {
                kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber,
                kUnknownOldestAncesterTime, kUnknownFileCreationTime,
                300 /* epoch_number */, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
+               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0);
 
   edit.SetComparatorName("foo");
   edit.SetLogNumber(kBig + 100);
@@ -208,7 +208,7 @@ TEST_F(VersionEditTest, EncodeEmptyFile) {
                Temperature::kUnknown, kInvalidBlobFileNumber,
                kUnknownOldestAncesterTime, kUnknownFileCreationTime,
                1 /*epoch_number*/, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
+               kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0);
   std::string buffer;
   ASSERT_TRUE(!edit.EncodeTo(&buffer));
 }