@nxtedition/rocksdb 8.2.0-alpha.1 → 8.2.0

package/deps/rocksdb/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h

@@ -32,9 +32,8 @@ namespace ROCKSDB_NAMESPACE {
 DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
 (const ReadOptions& options, const MultiGetRange* batch,
  const autovector<BlockHandle, MultiGetContext::MAX_BATCH_SIZE>* handles,
- autovector<Status, MultiGetContext::MAX_BATCH_SIZE>* statuses,
- autovector<CachableEntry<Block>, MultiGetContext::MAX_BATCH_SIZE>* results,
- char* scratch, const UncompressionDict& uncompression_dict) const {
+ Status* statuses, CachableEntry<Block>* results, char* scratch,
+ const UncompressionDict& uncompression_dict) const {
   RandomAccessFileReader* file = rep_->file.get();
   const Footer& footer = rep_->footer;
   const ImmutableOptions& ioptions = rep_->ioptions;
@@ -52,13 +51,14 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
         continue;
       }
 
-      (*statuses)[idx_in_batch] =
+      statuses[idx_in_batch] =
           RetrieveBlock(nullptr, options, handle, uncompression_dict,
-                        &(*results)[idx_in_batch].As<Block_kData>(),
+                        &results[idx_in_batch].As<Block_kData>(),
                         mget_iter->get_context, &lookup_data_block_context,
                         /* for_compaction */ false, /* use_cache */ true,
-                        /* wait_for_cache */ true, /* async_read */ false);
+                        /* async_read */ false);
     }
+    assert(idx_in_batch == handles->size());
     CO_RETURN;
   }
 
@@ -261,12 +261,12 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
       if (options.fill_cache) {
         BlockCacheLookupContext lookup_data_block_context(
             TableReaderCaller::kUserMultiGet);
-        CachableEntry<Block>* block_entry = &(*results)[idx_in_batch];
+        CachableEntry<Block>* block_entry = &results[idx_in_batch];
         // MaybeReadBlockAndLoadToCache will insert into the block caches if
         // necessary. Since we're passing the serialized block contents, it
         // will avoid looking up the block cache
         s = MaybeReadBlockAndLoadToCache(
-            nullptr, options, handle, uncompression_dict, /*wait=*/true,
+            nullptr, options, handle, uncompression_dict,
             /*for_compaction=*/false, &block_entry->As<Block_kData>(),
             mget_iter->get_context, &lookup_data_block_context,
             &serialized_block, /*async_read=*/false);
@@ -301,11 +301,11 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
         contents = std::move(serialized_block);
       }
       if (s.ok()) {
-        (*results)[idx_in_batch].SetOwnedValue(std::make_unique<Block>(
+        results[idx_in_batch].SetOwnedValue(std::make_unique<Block>(
             std::move(contents), read_amp_bytes_per_bit, ioptions.stats));
       }
     }
-    (*statuses)[idx_in_batch] = s;
+    statuses[idx_in_batch] = s;
   }
 }
 
@@ -355,152 +355,147 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::MultiGet)
 
     uint64_t prev_offset = std::numeric_limits<uint64_t>::max();
     autovector<BlockHandle, MultiGetContext::MAX_BATCH_SIZE> block_handles;
-    autovector<CachableEntry<Block>, MultiGetContext::MAX_BATCH_SIZE> results;
-    autovector<Status, MultiGetContext::MAX_BATCH_SIZE> statuses;
+    std::array<CachableEntry<Block>, MultiGetContext::MAX_BATCH_SIZE> results;
+    std::array<Status, MultiGetContext::MAX_BATCH_SIZE> statuses;
     MultiGetContext::Mask reused_mask = 0;
     char stack_buf[kMultiGetReadStackBufSize];
     std::unique_ptr<char[]> block_buf;
     {
       MultiGetRange data_block_range(sst_file_range, sst_file_range.begin(),
                                      sst_file_range.end());
-      std::vector<Cache::Handle*> cache_handles;
-      bool wait_for_cache_results = false;
-
       CachableEntry<UncompressionDict> uncompression_dict;
       Status uncompression_dict_status;
       uncompression_dict_status.PermitUncheckedError();
       bool uncompression_dict_inited = false;
       size_t total_len = 0;
-      ReadOptions ro = read_options;
-      ro.read_tier = kBlockCacheTier;
-
-      for (auto miter = data_block_range.begin();
-           miter != data_block_range.end(); ++miter) {
-        const Slice& key = miter->ikey;
-        iiter->Seek(miter->ikey);
 
-        IndexValue v;
-        if (iiter->Valid()) {
-          v = iiter->value();
-        }
-        if (!iiter->Valid() ||
-            (!v.first_internal_key.empty() && !skip_filters &&
-             UserComparatorWrapper(rep_->internal_comparator.user_comparator())
-                     .CompareWithoutTimestamp(
-                         ExtractUserKey(key),
-                         ExtractUserKey(v.first_internal_key)) < 0)) {
-          // The requested key falls between highest key in previous block and
-          // lowest key in current block.
-          if (!iiter->status().IsNotFound()) {
-            *(miter->s) = iiter->status();
+      // GetContext for any key will do, as the stats will be aggregated
+      // anyway
+      GetContext* get_context = sst_file_range.begin()->get_context;
+
+      {
+        using BCI = BlockCacheInterface<Block_kData>;
+        BCI block_cache{rep_->table_options.block_cache.get()};
+        std::array<BCI::TypedAsyncLookupHandle, MultiGetContext::MAX_BATCH_SIZE>
+            async_handles;
+        std::array<CacheKey, MultiGetContext::MAX_BATCH_SIZE> cache_keys;
+        size_t cache_lookup_count = 0;
+
+        for (auto miter = data_block_range.begin();
+             miter != data_block_range.end(); ++miter) {
+          const Slice& key = miter->ikey;
+          iiter->Seek(miter->ikey);
+
+          IndexValue v;
+          if (iiter->Valid()) {
+            v = iiter->value();
+          }
+          if (!iiter->Valid() ||
+              (!v.first_internal_key.empty() && !skip_filters &&
+               UserComparatorWrapper(
+                   rep_->internal_comparator.user_comparator())
+                       .CompareWithoutTimestamp(
+                           ExtractUserKey(key),
+                           ExtractUserKey(v.first_internal_key)) < 0)) {
+            // The requested key falls between highest key in previous block and
+            // lowest key in current block.
+            if (!iiter->status().IsNotFound()) {
+              *(miter->s) = iiter->status();
+            }
+            data_block_range.SkipKey(miter);
+            sst_file_range.SkipKey(miter);
+            continue;
           }
-          data_block_range.SkipKey(miter);
-          sst_file_range.SkipKey(miter);
-          continue;
-        }
 
-        if (!uncompression_dict_inited && rep_->uncompression_dict_reader) {
-          uncompression_dict_status =
-              rep_->uncompression_dict_reader->GetOrReadUncompressionDictionary(
-                  nullptr /* prefetch_buffer */, no_io,
-                  read_options.verify_checksums,
-                  sst_file_range.begin()->get_context, &lookup_context,
-                  &uncompression_dict);
-          uncompression_dict_inited = true;
-        }
+          if (!uncompression_dict_inited && rep_->uncompression_dict_reader) {
+            uncompression_dict_status =
+                rep_->uncompression_dict_reader
+                    ->GetOrReadUncompressionDictionary(
+                        nullptr /* prefetch_buffer */, no_io,
+                        read_options.verify_checksums, get_context,
+                        &lookup_context, &uncompression_dict);
+            uncompression_dict_inited = true;
+          }
 
-        if (!uncompression_dict_status.ok()) {
-          assert(!uncompression_dict_status.IsNotFound());
-          *(miter->s) = uncompression_dict_status;
-          data_block_range.SkipKey(miter);
-          sst_file_range.SkipKey(miter);
-          continue;
-        }
+          if (!uncompression_dict_status.ok()) {
+            assert(!uncompression_dict_status.IsNotFound());
+            *(miter->s) = uncompression_dict_status;
+            data_block_range.SkipKey(miter);
+            sst_file_range.SkipKey(miter);
+            continue;
+          }
 
-        statuses.emplace_back();
-        results.emplace_back();
-        if (v.handle.offset() == prev_offset) {
-          // This key can reuse the previous block (later on).
-          // Mark previous as "reused"
-          reused_mask |= MultiGetContext::Mask{1} << (block_handles.size() - 1);
-          // Use null handle to indicate this one reuses same block as
-          // previous.
-          block_handles.emplace_back(BlockHandle::NullBlockHandle());
-          continue;
-        }
-        // Lookup the cache for the given data block referenced by an index
-        // iterator value (i.e BlockHandle). If it exists in the cache,
-        // initialize block to the contents of the data block.
-        prev_offset = v.handle.offset();
-        BlockHandle handle = v.handle;
-        BlockCacheLookupContext lookup_data_block_context(
-            TableReaderCaller::kUserMultiGet);
-        const UncompressionDict& dict = uncompression_dict.GetValue()
-                                            ? *uncompression_dict.GetValue()
-                                            : UncompressionDict::GetEmptyDict();
-        Status s = RetrieveBlock(
-            nullptr, ro, handle, dict, &(results.back()).As<Block_kData>(),
-            miter->get_context, &lookup_data_block_context,
-            /* for_compaction */ false, /* use_cache */ true,
-            /* wait_for_cache */ false, /* async_read */ false);
-        if (s.IsIncomplete()) {
-          s = Status::OK();
-        }
-        if (s.ok() && !results.back().IsEmpty()) {
-          // Since we have a valid handle, check the value. If its nullptr,
-          // it means the cache is waiting for the final result and we're
-          // supposed to call WaitAll() to wait for the result.
-          if (results.back().GetValue() != nullptr) {
-            // Found it in the cache. Add NULL handle to indicate there is
-            // nothing to read from disk.
-            if (results.back().GetCacheHandle()) {
-              results.back().UpdateCachedValue();
-            }
+          if (v.handle.offset() == prev_offset) {
+            // This key can reuse the previous block (later on).
+            // Mark previous as "reused"
+            reused_mask |= MultiGetContext::Mask{1}
+                           << (block_handles.size() - 1);
+            // Use null handle to indicate this one reuses same block as
+            // previous.
             block_handles.emplace_back(BlockHandle::NullBlockHandle());
-          } else {
-            // We have to wait for the cache lookup to finish in the
-            // background, and then we may have to read the block from disk
-            // anyway
-            assert(results.back().GetCacheHandle());
-            wait_for_cache_results = true;
-            block_handles.emplace_back(handle);
-            cache_handles.emplace_back(results.back().GetCacheHandle());
+            continue;
+          }
+          prev_offset = v.handle.offset();
+          block_handles.emplace_back(v.handle);
+
+          if (block_cache) {
+            // Lookup the cache for the given data block referenced by an index
+            // iterator value (i.e BlockHandle). If it exists in the cache,
+            // initialize block to the contents of the data block.
+            // TODO?
+            // BlockCacheLookupContext lookup_data_block_context(
+            //    TableReaderCaller::kUserMultiGet);
+
+            // An async version of MaybeReadBlockAndLoadToCache /
+            // GetDataBlockFromCache
+            BCI::TypedAsyncLookupHandle& async_handle =
+                async_handles[cache_lookup_count];
+            cache_keys[cache_lookup_count] =
+                GetCacheKey(rep_->base_cache_key, v.handle);
+            async_handle.key = cache_keys[cache_lookup_count].AsSlice();
+            // NB: StartAsyncLookupFull populates async_handle.helper
+            async_handle.create_context = &rep_->create_context;
+            async_handle.priority = GetCachePriority<Block_kData>();
+            async_handle.stats = rep_->ioptions.statistics.get();
+
+            block_cache.StartAsyncLookupFull(
+                async_handle, rep_->ioptions.lowest_used_cache_tier);
+            ++cache_lookup_count;
+            // TODO: stats?
           }
-        } else {
-          block_handles.emplace_back(handle);
-          total_len += BlockSizeWithTrailer(handle);
         }
-      }
 
-      if (wait_for_cache_results) {
-        Cache* block_cache = rep_->table_options.block_cache.get();
-        block_cache->WaitAll(cache_handles);
+        if (block_cache) {
+          block_cache.get()->WaitAll(&async_handles[0], cache_lookup_count);
+        }
+        size_t lookup_idx = 0;
         for (size_t i = 0; i < block_handles.size(); ++i) {
           // If this block was a success or failure or not needed because
           // the corresponding key is in the same block as a prior key, skip
-          if (block_handles[i] == BlockHandle::NullBlockHandle() ||
-              results[i].IsEmpty()) {
+          if (block_handles[i] == BlockHandle::NullBlockHandle()) {
             continue;
           }
-          results[i].UpdateCachedValue();
-          void* val = results[i].GetValue();
-          Cache::Handle* handle = results[i].GetCacheHandle();
-          // GetContext for any key will do, as the stats will be aggregated
-          // anyway
-          GetContext* get_context = sst_file_range.begin()->get_context;
-          if (!val) {
-            // The async cache lookup failed - could be due to an error
-            // or a false positive. We need to read the data block from
-            // the SST file
-            results[i].Reset();
+          if (!block_cache) {
             total_len += BlockSizeWithTrailer(block_handles[i]);
-            UpdateCacheMissMetrics(BlockType::kData, get_context);
           } else {
-            block_handles[i] = BlockHandle::NullBlockHandle();
-            UpdateCacheHitMetrics(BlockType::kData, get_context,
-                                  block_cache->GetUsage(handle));
+            BCI::TypedHandle* h = async_handles[lookup_idx].Result();
+            if (h) {
+              // Cache hit
+              results[i].SetCachedValue(block_cache.Value(h), block_cache.get(),
+                                        h);
+              // Don't need to fetch
+              block_handles[i] = BlockHandle::NullBlockHandle();
+              UpdateCacheHitMetrics(BlockType::kData, get_context,
+                                    block_cache.get()->GetUsage(h));
+            } else {
+              // Cache miss
+              total_len += BlockSizeWithTrailer(block_handles[i]);
+              UpdateCacheMissMetrics(BlockType::kData, get_context);
+            }
+            ++lookup_idx;
           }
         }
+        assert(lookup_idx == cache_lookup_count);
       }
 
       if (total_len) {
@@ -530,11 +525,10 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::MultiGet)
           }
         }
         CO_AWAIT(RetrieveMultipleBlocks)
-        (read_options, &data_block_range, &block_handles, &statuses, &results,
-         scratch, dict);
-        if (sst_file_range.begin()->get_context) {
-          ++(sst_file_range.begin()
-                 ->get_context->get_context_stats_.num_sst_read);
+        (read_options, &data_block_range, &block_handles, &statuses[0],
+         &results[0], scratch, dict);
+        if (get_context) {
+          ++(get_context->get_context_stats_.num_sst_read);
         }
       }
     }

package/deps/rocksdb/rocksdb/table/block_based/block_cache.cc

@@ -50,36 +50,36 @@ namespace {
 // For getting SecondaryCache-compatible helpers from a BlockType. This is
 // useful for accessing block cache in untyped contexts, such as for generic
 // cache warming in table builder.
-constexpr std::array<const Cache::CacheItemHelper*,
-                     static_cast<unsigned>(BlockType::kInvalid) + 1>
+const std::array<const Cache::CacheItemHelper*,
+                 static_cast<unsigned>(BlockType::kInvalid) + 1>
     kCacheItemFullHelperForBlockType{{
-        &BlockCacheInterface<Block_kData>::kFullHelper,
-        &BlockCacheInterface<ParsedFullFilterBlock>::kFullHelper,
-        &BlockCacheInterface<Block_kFilterPartitionIndex>::kFullHelper,
+        BlockCacheInterface<Block_kData>::GetFullHelper(),
+        BlockCacheInterface<ParsedFullFilterBlock>::GetFullHelper(),
+        BlockCacheInterface<Block_kFilterPartitionIndex>::GetFullHelper(),
         nullptr,  // kProperties
-        &BlockCacheInterface<UncompressionDict>::kFullHelper,
-        &BlockCacheInterface<Block_kRangeDeletion>::kFullHelper,
+        BlockCacheInterface<UncompressionDict>::GetFullHelper(),
+        BlockCacheInterface<Block_kRangeDeletion>::GetFullHelper(),
         nullptr,  // kHashIndexPrefixes
         nullptr,  // kHashIndexMetadata
         nullptr,  // kMetaIndex (not yet stored in block cache)
-        &BlockCacheInterface<Block_kIndex>::kFullHelper,
+        BlockCacheInterface<Block_kIndex>::GetFullHelper(),
         nullptr,  // kInvalid
     }};
 
 // For getting basic helpers from a BlockType (no SecondaryCache support)
-constexpr std::array<const Cache::CacheItemHelper*,
-                     static_cast<unsigned>(BlockType::kInvalid) + 1>
+const std::array<const Cache::CacheItemHelper*,
+                 static_cast<unsigned>(BlockType::kInvalid) + 1>
     kCacheItemBasicHelperForBlockType{{
-        &BlockCacheInterface<Block_kData>::kBasicHelper,
-        &BlockCacheInterface<ParsedFullFilterBlock>::kBasicHelper,
-        &BlockCacheInterface<Block_kFilterPartitionIndex>::kBasicHelper,
+        BlockCacheInterface<Block_kData>::GetBasicHelper(),
+        BlockCacheInterface<ParsedFullFilterBlock>::GetBasicHelper(),
+        BlockCacheInterface<Block_kFilterPartitionIndex>::GetBasicHelper(),
         nullptr,  // kProperties
-        &BlockCacheInterface<UncompressionDict>::kBasicHelper,
-        &BlockCacheInterface<Block_kRangeDeletion>::kBasicHelper,
+        BlockCacheInterface<UncompressionDict>::GetBasicHelper(),
+        BlockCacheInterface<Block_kRangeDeletion>::GetBasicHelper(),
         nullptr,  // kHashIndexPrefixes
         nullptr,  // kHashIndexMetadata
         nullptr,  // kMetaIndex (not yet stored in block cache)
-        &BlockCacheInterface<Block_kIndex>::kBasicHelper,
+        BlockCacheInterface<Block_kIndex>::GetBasicHelper(),
         nullptr,  // kInvalid
     }};
 }  // namespace

package/deps/rocksdb/rocksdb/table/block_based/cachable_entry.h

@@ -176,22 +176,6 @@ class CachableEntry {
     assert(!own_value_);
   }
 
-  void UpdateCachedValue() {
-    assert(cache_ != nullptr);
-    assert(cache_handle_ != nullptr);
-
-    value_ = static_cast<T*>(cache_->Value(cache_handle_));
-  }
-
-  bool IsReady() {
-    if (!own_value_) {
-      assert(cache_ != nullptr);
-      assert(cache_handle_ != nullptr);
-      return cache_->IsReady(cache_handle_);
-    }
-    return true;
-  }
-
   // Since this class is essentially an elaborate pointer, it's sometimes
   // useful to be able to upcast or downcast the base type of the pointer,
   // especially when interacting with typed_cache.h.

package/deps/rocksdb/rocksdb/table/block_based/filter_block_reader_common.cc

@@ -33,7 +33,7 @@ Status FilterBlockReaderCommon<TBlocklike>::ReadFilterBlock(
                            UncompressionDict::GetEmptyDict(), filter_block,
                            get_context, lookup_context,
                            /* for_compaction */ false, use_cache,
-                           /* wait_for_cache */ true, /* async_read */ false);
+                           /* async_read */ false);
 
   return s;
 }

package/deps/rocksdb/rocksdb/table/block_based/index_reader_common.cc

@@ -29,7 +29,7 @@ Status BlockBasedTable::IndexReaderCommon::ReadIndexBlock(
       prefetch_buffer, read_options, rep->footer.index_handle(),
       UncompressionDict::GetEmptyDict(), &index_block->As<Block_kIndex>(),
       get_context, lookup_context, /* for_compaction */ false, use_cache,
-      /* wait_for_cache */ true, /* async_read */ false);
+      /* async_read */ false);
 
   return s;
 }

package/deps/rocksdb/rocksdb/table/block_based/partitioned_filter_block.cc

@@ -318,7 +318,7 @@ Status PartitionedFilterBlockReader::GetFilterPartitionBlock(
                              UncompressionDict::GetEmptyDict(), filter_block,
                              get_context, lookup_context,
                              /* for_compaction */ false, /* use_cache */ true,
-                             /* wait_for_cache */ true, /* async_read */ false);
+                             /* async_read */ false);
 
   return s;
 }
@@ -518,9 +518,8 @@ Status PartitionedFilterBlockReader::CacheDependencies(const ReadOptions& ro,
     // filter blocks
     s = table()->MaybeReadBlockAndLoadToCache(
         prefetch_buffer.get(), ro, handle, UncompressionDict::GetEmptyDict(),
-        /* wait */ true, /* for_compaction */ false, &block,
-        nullptr /* get_context */, &lookup_context, nullptr /* contents */,
-        false);
+        /* for_compaction */ false, &block, nullptr /* get_context */,
+        &lookup_context, nullptr /* contents */, false);
     if (!s.ok()) {
       return s;
     }

package/deps/rocksdb/rocksdb/table/block_based/partitioned_index_reader.cc

@@ -192,7 +192,7 @@ Status PartitionIndexReader::CacheDependencies(const ReadOptions& ro,
     // filter blocks
     Status s = table()->MaybeReadBlockAndLoadToCache(
         prefetch_buffer.get(), ro, handle, UncompressionDict::GetEmptyDict(),
-        /*wait=*/true, /*for_compaction=*/false, &block.As<Block_kIndex>(),
+        /*for_compaction=*/false, &block.As<Block_kIndex>(),
         /*get_context=*/nullptr, &lookup_context, /*contents=*/nullptr,
         /*async_read=*/false);
 

package/deps/rocksdb/rocksdb/table/block_based/uncompression_dict_reader.cc

@@ -62,7 +62,7 @@ Status UncompressionDictReader::ReadUncompressionDictionary(
       prefetch_buffer, read_options, rep->compression_dict_handle,
       UncompressionDict::GetEmptyDict(), uncompression_dict, get_context,
       lookup_context,
-      /* for_compaction */ false, use_cache, /* wait_for_cache */ true,
+      /* for_compaction */ false, use_cache,
       /* async_read */ false);
 
   if (!s.ok()) {