@nxtedition/rocksdb 13.5.7 → 13.5.9

package/deps/rocksdb/rocksdb/table/sst_file_reader.cc

@@ -3,7 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-
 #include "rocksdb/sst_file_reader.h"
 
 #include "db/arena_wrapped_db_iter.h"
@@ -63,7 +62,8 @@ Status SstFileReader::Open(const std::string& file_path) {
   }
   if (s.ok()) {
     TableReaderOptions t_opt(
-        r->ioptions, r->moptions.prefix_extractor, r->soptions,
+        r->ioptions, r->moptions.prefix_extractor,
+        r->moptions.compression_manager.get(), r->soptions,
         r->ioptions.internal_comparator,
         r->moptions.block_protection_bytes_per_key,
         /*skip_filters*/ false, /*immortal*/ false,
@@ -81,6 +81,85 @@ Status SstFileReader::Open(const std::string& file_path) {
   return s;
 }
 
+std::vector<Status> SstFileReader::MultiGet(const ReadOptions& roptions,
+                                            const std::vector<Slice>& keys,
+                                            std::vector<std::string>* values) {
+  const auto num_keys = keys.size();
+  std::vector<Status> statuses(num_keys, Status::OK());
+  std::vector<PinnableSlice> pin_values(num_keys);
+
+  auto r = rep_.get();
+  const Comparator* user_comparator =
+      r->ioptions.internal_comparator.user_comparator();
+
+  autovector<KeyContext, MultiGetContext::MAX_BATCH_SIZE> key_context;
+  autovector<KeyContext*, MultiGetContext::MAX_BATCH_SIZE> sorted_keys;
+  autovector<GetContext, MultiGetContext::MAX_BATCH_SIZE> get_ctx;
+  autovector<MergeContext, MultiGetContext::MAX_BATCH_SIZE> merge_ctx;
+  sorted_keys.resize(num_keys);
+  for (size_t i = 0; i < num_keys; ++i) {
+    PinnableSlice* val = &pin_values[i];
+    val->Reset();
+    merge_ctx.emplace_back();
+    key_context.emplace_back(nullptr, keys[i], val, nullptr,
+                             nullptr /* timestamp */, &statuses[i]);
+    get_ctx.emplace_back(user_comparator, r->ioptions.merge_operator.get(),
+                         nullptr, nullptr, GetContext::kNotFound,
+                         *key_context[i].key, val, nullptr, nullptr, nullptr,
+                         &merge_ctx[i], true,
+                         &key_context[i].max_covering_tombstone_seq, nullptr);
+    key_context[i].get_context = &get_ctx[i];
+  }
+  for (size_t i = 0; i < num_keys; ++i) {
+    sorted_keys[i] = &key_context[i];
+  }
+
+  struct CompareKeyContext {
+    explicit CompareKeyContext(const Comparator* comp) : comparator(comp) {}
+    inline bool operator()(const KeyContext* lhs, const KeyContext* rhs) const {
+      return comparator->CompareWithoutTimestamp(*(lhs->key), false,
+                                                 *(rhs->key), false) < 0;
+    }
+    const Comparator* comparator;
+  };
+
+  std::sort(sorted_keys.begin(), sorted_keys.end(),
+            CompareKeyContext(user_comparator));
+  const auto sequence = roptions.snapshot != nullptr
+                            ? roptions.snapshot->GetSequenceNumber()
+                            : kMaxSequenceNumber;
+  MultiGetContext ctx(&sorted_keys, 0, num_keys, sequence, roptions,
+                      r->ioptions.fs.get(), nullptr);
+  MultiGetRange range = ctx.GetMultiGetRange();
+  r->table_reader->MultiGet(roptions, &range,
+                            r->moptions.prefix_extractor.get(),
+                            false /* skip filters */);
+
+  values->resize(num_keys);
+  for (size_t i = 0; i < num_keys; ++i) {
+    if (statuses[i].ok()) {
+      switch (get_ctx[i].State()) {
+        case GetContext::kFound:
+          (*values)[i].assign(pin_values[i].data(), pin_values[i].size());
+          break;
+        case GetContext::kNotFound:
+        case GetContext::kDeleted:
+          statuses[i] = Status::NotFound();
+          break;
+        case GetContext::kMerge:
+          statuses[i] = Status::MergeInProgress();
+          break;
+        case GetContext::kCorrupt:
+        case GetContext::kUnexpectedBlobIndex:
+        case GetContext::kMergeOperatorFailed:
+          statuses[i] = Status::Corruption();
+          break;
+      };
+    }
+  }
+  return statuses;
+}
+
 Iterator* SstFileReader::NewIterator(const ReadOptions& roptions) {
   assert(roptions.io_activity == Env::IOActivity::kUnknown);
   auto r = rep_.get();
@@ -88,11 +167,11 @@ Iterator* SstFileReader::NewIterator(const ReadOptions& roptions) {
                       ? roptions.snapshot->GetSequenceNumber()
                       : kMaxSequenceNumber;
   ArenaWrappedDBIter* res = new ArenaWrappedDBIter();
-  res->Init(
-      r->options.env, roptions, r->ioptions, r->moptions, nullptr /* version */,
-      sequence, r->moptions.max_sequential_skip_in_iterations,
-      0 /* version_number */, nullptr /* read_callback */, nullptr /* cfh */,
-      true /* expose_blob_index */, false /* allow_refresh */);
+  res->Init(r->options.env, roptions, r->ioptions, r->moptions,
+            nullptr /* version */, sequence, 0 /* version_number */,
+            nullptr /* read_callback */, nullptr /* cfh */,
+            true /* expose_blob_index */, false /* allow_refresh */,
+            /*active_mem=*/nullptr);
   auto internal_iter = r->table_reader->NewIterator(
       res->GetReadOptions(), r->moptions.prefix_extractor.get(),
       res->GetArena(), false /* skip_filters */,

package/deps/rocksdb/rocksdb/table/sst_file_reader_test.cc

@@ -3,7 +3,6 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
-
 #include "rocksdb/sst_file_reader.h"
 
 #include <cinttypes>
@@ -769,6 +768,94 @@ TEST_F(SstFileReaderTableIteratorTest, UserDefinedTimestampsEnabled) {
   Close();
 }
 
+class SstFileReaderTableMultiGetTest : public DBTestBase {
+ public:
+  SstFileReaderTableMultiGetTest()
+      : DBTestBase("sst_file_reader_table_multi_get_test",
+                   /*env_do_fsync=*/false) {}
+
+  void VerifyTableEntry(Iterator* iter, const std::string& user_key,
+                        ValueType value_type,
+                        std::optional<std::string> expected_value,
+                        bool backward_iteration = false) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_TRUE(iter->status().ok());
+    ParsedInternalKey pikey;
+    ASSERT_OK(ParseInternalKey(iter->key(), &pikey, /*log_err_key=*/false));
+    ASSERT_EQ(pikey.user_key, user_key);
+    ASSERT_EQ(pikey.type, value_type);
+    if (expected_value.has_value()) {
+      ASSERT_EQ(iter->value(), expected_value.value());
+    }
+    if (!backward_iteration) {
+      iter->Next();
+    } else {
+      iter->Prev();
+    }
+  }
+};
+
+TEST_F(SstFileReaderTableMultiGetTest, Basic) {
+  Options options = CurrentOptions();
+  const Comparator* ucmp = BytewiseComparator();
+  options.comparator = ucmp;
+  options.disable_auto_compactions = true;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  BlockBasedTableOptions bbto;
+  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+
+  DestroyAndReopen(options);
+
+  ASSERT_OK(Put("foo", "val1"));
+  const Snapshot* snapshot1 = dbfull()->GetSnapshot();
+  ASSERT_OK(Delete("foo"));
+  ASSERT_OK(Delete("bar"));
+  const Snapshot* snapshot2 = dbfull()->GetSnapshot();
+  ASSERT_OK(Put("bar", "val2"));
+  ASSERT_OK(Put("baz", "val3"));
+  ASSERT_OK(Put("aaa", "val4"));
+  const Snapshot* snapshot3 = dbfull()->GetSnapshot();
+  ASSERT_OK(Merge("aaa", "val5"));
+
+  ASSERT_OK(Flush());
+
+  std::vector<LiveFileMetaData> files;
+  dbfull()->GetLiveFilesMetaData(&files);
+  ASSERT_TRUE(files.size() == 1);
+  ASSERT_TRUE(files[0].level == 0);
+  std::string file_name = files[0].directory + "/" + files[0].relative_filename;
+
+  SstFileReader reader(options);
+  ASSERT_OK(reader.Open(file_name));
+  ASSERT_OK(reader.VerifyChecksum());
+
+  std::vector<Slice> keys;
+  std::vector<std::string> values;
+
+  keys.emplace_back("fo1");
+  keys.emplace_back("foo");
+  keys.emplace_back("baz");
+  keys.emplace_back("bar");
+  keys.emplace_back("aaa");
+  auto statuses = reader.MultiGet(ReadOptions(), keys, &values);
+  ASSERT_TRUE(statuses[0].IsNotFound())
+      << "Failed: status=" << statuses[0].ToString() << " val=" << values[0];
+  ASSERT_TRUE(statuses[1].IsNotFound())
+      << "Failed: status=" << statuses[0].ToString() << " val=" << values[1];
+  ASSERT_TRUE(statuses[2].ok());
+  ASSERT_TRUE(statuses[3].ok());
+  ASSERT_EQ("val3", values[2]);
+  ASSERT_EQ("val2", values[3]);
+  ASSERT_TRUE(statuses[4].ok());
+  ASSERT_EQ("val4,val5", values[4]);
+
+  dbfull()->ReleaseSnapshot(snapshot1);
+  dbfull()->ReleaseSnapshot(snapshot2);
+  dbfull()->ReleaseSnapshot(snapshot3);
+  Close();
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
@@ -777,4 +864,3 @@ int main(int argc, char** argv) {
   RegisterCustomObjects(argc, argv);
   return RUN_ALL_TESTS();
 }
-

package/deps/rocksdb/rocksdb/table/sst_file_writer.cc

@@ -25,7 +25,6 @@ const std::string ExternalSstFilePropertyNames::kVersion =
 const std::string ExternalSstFilePropertyNames::kGlobalSeqno =
     "rocksdb.external_sst_file.global_seqno";
 
-
 const size_t kFadviseTrigger = 1024 * 1024;  // 1MB
 
 struct SstFileWriter::Rep {

package/deps/rocksdb/rocksdb/table/table_builder.h

@@ -24,6 +24,7 @@
 #include "rocksdb/table_properties.h"
 #include "table/unique_id_impl.h"
 #include "trace_replay/block_cache_tracer.h"
+#include "util/cast_util.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -35,6 +36,7 @@ struct TableReaderOptions {
   TableReaderOptions(
       const ImmutableOptions& _ioptions,
       const std::shared_ptr<const SliceTransform>& _prefix_extractor,
+      UnownedPtr<CompressionManager> _compression_manager,
       const EnvOptions& _env_options,
       const InternalKeyComparator& _internal_comparator,
       uint8_t _block_protection_bytes_per_key, bool _skip_filters = false,
@@ -46,6 +48,7 @@ struct TableReaderOptions {
       uint64_t _tail_size = 0, bool _user_defined_timestamps_persisted = true)
       : ioptions(_ioptions),
         prefix_extractor(_prefix_extractor),
+        compression_manager(_compression_manager),
         env_options(_env_options),
         internal_comparator(_internal_comparator),
         skip_filters(_skip_filters),
@@ -64,6 +67,9 @@ struct TableReaderOptions {
 
   const ImmutableOptions& ioptions;
   const std::shared_ptr<const SliceTransform>& prefix_extractor;
+  // NOTE: the compression manager is not saved, just potentially a decompressor
+  // from it, so we don't need a shared_ptr copy
+  UnownedPtr<CompressionManager> compression_manager;
   const EnvOptions& env_options;
   const InternalKeyComparator& internal_comparator;
   // This is only used for BlockBasedTable (reader)
@@ -207,6 +213,9 @@ class TableBuilder {
     return NumEntries() == 0 && GetTableProperties().num_range_deletions == 0;
   }
 
+  // Size of the file before its content is compressed.
+  virtual uint64_t PreCompressionSize() const { return 0; }
+
   // Size of the file generated so far.  If invoked after a successful
   // Finish() call, returns the size of the final generated file.
   virtual uint64_t FileSize() const = 0;
@@ -235,7 +244,7 @@ class TableBuilder {
   // enforced state (ready to encode to string).
   virtual void SetSeqnoTimeTableProperties(
       const SeqnoToTimeMapping& /*relevant_mapping*/,
-      uint64_t /*oldest_ancestor_time*/){}
+      uint64_t /*oldest_ancestor_time*/) {}
 };
 
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/table/table_reader_bench.cc

@@ -145,8 +145,9 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
     std::unique_ptr<RandomAccessFileReader> file_reader(
         new RandomAccessFileReader(std::move(raf), file_name));
     s = opts.table_factory->NewTableReader(
-        TableReaderOptions(ioptions, moptions.prefix_extractor, env_options,
-                           ikc, 0 /* block_protection_bytes_per_key */),
+        TableReaderOptions(ioptions, moptions.prefix_extractor,
+                           moptions.compression_manager.get(), env_options, ikc,
+                           0 /* block_protection_bytes_per_key */),
         std::move(file_reader), file_size, &table_reader);
     if (!s.ok()) {
       fprintf(stderr, "Open Table Error: %s\n", s.ToString().c_str());