@nxtedition/rocksdb 7.1.20 → 7.1.21

package/deps/rocksdb/rocksdb/db/wide/wide_column_serialization_test.cc

@@ -124,6 +124,25 @@ TEST(WideColumnSerializationTest, SerializeDeserialize) {
   }
 }
 
+TEST(WideColumnSerializationTest, SerializeWithPrepend) {
+  Slice value_of_default("baz");
+  WideColumns other_columns{{"foo", "bar"}, {"hello", "world"}};
+
+  std::string output;
+  ASSERT_OK(WideColumnSerialization::Serialize(value_of_default, other_columns,
+                                               output));
+
+  Slice input(output);
+
+  WideColumns deserialized_columns;
+  ASSERT_OK(WideColumnSerialization::Deserialize(input, deserialized_columns));
+
+  WideColumns expected_columns{{kDefaultWideColumnName, value_of_default},
+                               other_columns[0],
+                               other_columns[1]};
+  ASSERT_EQ(deserialized_columns, expected_columns);
+}
+
 TEST(WideColumnSerializationTest, SerializeDuplicateError) {
   WideColumns columns{{"foo", "bar"}, {"foo", "baz"}};
   std::string output;
@@ -132,6 +151,16 @@ TEST(WideColumnSerializationTest, SerializeDuplicateError) {
       WideColumnSerialization::Serialize(columns, output).IsCorruption());
 }
 
+TEST(WideColumnSerializationTest, SerializeWithPrependDuplicateError) {
+  Slice value_of_default("baz");
+  WideColumns other_columns{{kDefaultWideColumnName, "dup"}, {"foo", "bar"}};
+
+  std::string output;
+  ASSERT_TRUE(WideColumnSerialization::Serialize(value_of_default,
+                                                 other_columns, output)
+                  .IsCorruption());
+}
+
 TEST(WideColumnSerializationTest, SerializeOutOfOrderError) {
   WideColumns columns{{"hello", "world"}, {"foo", "bar"}};
   std::string output;
@@ -303,6 +332,7 @@ TEST(WideColumnSerializationTest, DeserializeColumnsOutOfOrder) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

package/deps/rocksdb/rocksdb/db/wide/wide_columns.cc

@@ -9,6 +9,10 @@
 
 namespace ROCKSDB_NAMESPACE {
 
+const Slice kDefaultWideColumnName;
+
+const WideColumns kNoWideColumns;
+
 Status PinnableWideColumns::CreateIndexForWideColumns() {
   Slice value_copy = value_;
 

package/deps/rocksdb/rocksdb/db/write_batch.cc

@@ -1353,8 +1353,31 @@ Status WriteBatch::DeleteRange(ColumnFamilyHandle* column_family,
     return WriteBatchInternal::DeleteRange(this, cf_id, begin_key, end_key);
   }
 
-  return Status::InvalidArgument(
-      "Cannot call this method on column family enabling timestamp");
+  needs_in_place_update_ts_ = true;
+  has_key_with_ts_ = true;
+  std::string dummy_ts(ts_sz, '\0');
+  std::array<Slice, 2> begin_key_with_ts{{begin_key, dummy_ts}};
+  std::array<Slice, 2> end_key_with_ts{{end_key, dummy_ts}};
+  return WriteBatchInternal::DeleteRange(
+      this, cf_id, SliceParts(begin_key_with_ts.data(), 2),
+      SliceParts(end_key_with_ts.data(), 2));
+}
+
+Status WriteBatch::DeleteRange(ColumnFamilyHandle* column_family,
+                               const Slice& begin_key, const Slice& end_key,
+                               const Slice& ts) {
+  const Status s = CheckColumnFamilyTimestampSize(column_family, ts);
+  if (!s.ok()) {
+    return s;
+  }
+  assert(column_family);
+  has_key_with_ts_ = true;
+  uint32_t cf_id = column_family->GetID();
+  std::array<Slice, 2> key_with_ts{{begin_key, ts}};
+  std::array<Slice, 2> end_key_with_ts{{end_key, ts}};
+  return WriteBatchInternal::DeleteRange(this, cf_id,
+                                         SliceParts(key_with_ts.data(), 2),
+                                         SliceParts(end_key_with_ts.data(), 2));
 }
 
 Status WriteBatchInternal::DeleteRange(WriteBatch* b, uint32_t column_family_id,
@@ -1928,10 +1951,9 @@ class MemTableInserter : public WriteBatch::Handler {
       // always 0 in
       // non-recovery, regular write code-path)
       // * If recovering_log_number_ < cf_mems_->GetLogNumber(), this means that
-      // column
-      // family already contains updates from this log. We can't apply updates
-      // twice because of update-in-place or merge workloads -- ignore the
-      // update
+      // column family already contains updates from this log. We can't apply
+      // updates twice because of update-in-place or merge workloads -- ignore
+      // the update
       *s = Status::OK();
       return false;
     }
@@ -2331,7 +2353,8 @@ class MemTableInserter : public WriteBatch::Handler {
             cfd->ioptions()->table_factory->Name() + " in CF " +
             cfd->GetName());
       }
-      int cmp = cfd->user_comparator()->Compare(begin_key, end_key);
+      int cmp =
+          cfd->user_comparator()->CompareWithoutTimestamp(begin_key, end_key);
       if (cmp > 0) {
         // TODO(ajkr): refactor `SeekToColumnFamily()` so it returns a `Status`.
         ret_status.PermitUncheckedError();

package/deps/rocksdb/rocksdb/db/write_batch_internal.h

@@ -314,8 +314,12 @@ class TimestampUpdater : public WriteBatch::Handler {
   }
 
   Status DeleteRangeCF(uint32_t cf, const Slice& begin_key,
-                       const Slice&) override {
-    return UpdateTimestamp(cf, begin_key);
+                       const Slice& end_key) override {
+    Status s = UpdateTimestamp(cf, begin_key, true /* is_key */);
+    if (s.ok()) {
+      s = UpdateTimestamp(cf, end_key, false /* is_key */);
+    }
+    return s;
   }
 
   Status MergeCF(uint32_t cf, const Slice& key, const Slice&) override {
@@ -341,13 +345,15 @@ class TimestampUpdater : public WriteBatch::Handler {
   Status MarkNoop(bool /*empty_batch*/) override { return Status::OK(); }
 
  private:
-  Status UpdateTimestamp(uint32_t cf, const Slice& key) {
-    Status s = UpdateTimestampImpl(cf, key, idx_);
+  // @param is_key specifies whether the update is for key or value.
+  Status UpdateTimestamp(uint32_t cf, const Slice& buf, bool is_key = true) {
+    Status s = UpdateTimestampImpl(cf, buf, idx_, is_key);
     ++idx_;
     return s;
   }
 
-  Status UpdateTimestampImpl(uint32_t cf, const Slice& key, size_t /*idx*/) {
+  Status UpdateTimestampImpl(uint32_t cf, const Slice& buf, size_t /*idx*/,
+                             bool is_key) {
     if (timestamp_.empty()) {
       return Status::InvalidArgument("Timestamp is empty");
     }
@@ -361,22 +367,27 @@ class TimestampUpdater : public WriteBatch::Handler {
     } else if (cf_ts_sz != timestamp_.size()) {
       return Status::InvalidArgument("timestamp size mismatch");
     }
-    UpdateProtectionInformationIfNeeded(key, timestamp_);
+    UpdateProtectionInformationIfNeeded(buf, timestamp_, is_key);
 
-    char* ptr = const_cast<char*>(key.data() + key.size() - cf_ts_sz);
+    char* ptr = const_cast<char*>(buf.data() + buf.size() - cf_ts_sz);
     assert(ptr);
     memcpy(ptr, timestamp_.data(), timestamp_.size());
     return Status::OK();
   }
 
-  void UpdateProtectionInformationIfNeeded(const Slice& key, const Slice& ts) {
+  void UpdateProtectionInformationIfNeeded(const Slice& buf, const Slice& ts,
+                                           bool is_key) {
     if (prot_info_ != nullptr) {
       const size_t ts_sz = ts.size();
-      SliceParts old_key(&key, 1);
-      Slice key_no_ts(key.data(), key.size() - ts_sz);
-      std::array<Slice, 2> new_key_cmpts{{key_no_ts, ts}};
-      SliceParts new_key(new_key_cmpts.data(), 2);
-      prot_info_->entries_[idx_].UpdateK(old_key, new_key);
+      SliceParts old(&buf, 1);
+      Slice old_no_ts(buf.data(), buf.size() - ts_sz);
+      std::array<Slice, 2> new_key_cmpts{{old_no_ts, ts}};
+      SliceParts new_parts(new_key_cmpts.data(), 2);
+      if (is_key) {
+        prot_info_->entries_[idx_].UpdateK(old, new_parts);
+      } else {
+        prot_info_->entries_[idx_].UpdateV(old, new_parts);
+      }
     }
   }
 

package/deps/rocksdb/rocksdb/db/write_batch_test.cc

@@ -962,15 +962,15 @@ TEST_F(WriteBatchTest, SanityChecks) {
   ASSERT_TRUE(wb.Delete(nullptr, "key", "ts").IsInvalidArgument());
   ASSERT_TRUE(wb.SingleDelete(nullptr, "key", "ts").IsInvalidArgument());
   ASSERT_TRUE(wb.Merge(nullptr, "key", "ts", "value").IsNotSupported());
-  ASSERT_TRUE(
-      wb.DeleteRange(nullptr, "begin_key", "end_key", "ts").IsNotSupported());
+  ASSERT_TRUE(wb.DeleteRange(nullptr, "begin_key", "end_key", "ts")
+                  .IsInvalidArgument());
 
   ASSERT_TRUE(wb.Put(&cf4, "key", "ts", "value").IsInvalidArgument());
   ASSERT_TRUE(wb.Delete(&cf4, "key", "ts").IsInvalidArgument());
   ASSERT_TRUE(wb.SingleDelete(&cf4, "key", "ts").IsInvalidArgument());
   ASSERT_TRUE(wb.Merge(&cf4, "key", "ts", "value").IsNotSupported());
   ASSERT_TRUE(
-      wb.DeleteRange(&cf4, "begin_key", "end_key", "ts").IsNotSupported());
+      wb.DeleteRange(&cf4, "begin_key", "end_key", "ts").IsInvalidArgument());
 
   constexpr size_t wrong_ts_sz = 1 + sizeof(uint64_t);
   std::string ts(wrong_ts_sz, '\0');
@@ -980,7 +980,7 @@ TEST_F(WriteBatchTest, SanityChecks) {
   ASSERT_TRUE(wb.SingleDelete(&cf0, "key", ts).IsInvalidArgument());
   ASSERT_TRUE(wb.Merge(&cf0, "key", ts, "value").IsNotSupported());
   ASSERT_TRUE(
-      wb.DeleteRange(&cf0, "begin_key", "end_key", ts).IsNotSupported());
+      wb.DeleteRange(&cf0, "begin_key", "end_key", ts).IsInvalidArgument());
 
   // Sanity checks for the new WriteBatch APIs without extra 'ts' arg.
   WriteBatch wb1(0, 0, 0, wrong_ts_sz);
@@ -1107,6 +1107,7 @@ TEST_F(WriteBatchTest, CommitWithTimestamp) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

package/deps/rocksdb/rocksdb/db/write_callback_test.cc

@@ -447,6 +447,7 @@ TEST_F(WriteCallbackTest, WriteCallBackTest) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

package/deps/rocksdb/rocksdb/db/write_controller_test.cc

@@ -242,6 +242,7 @@ TEST_F(WriteControllerTest, CreditAccumulation) {
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

package/deps/rocksdb/rocksdb/db_stress_tool/batched_ops_stress.cc

@@ -26,31 +26,41 @@ class BatchedOpsStressTest : public StressTest {
                  const std::vector<int>& rand_column_families,
                  const std::vector<int64_t>& rand_keys,
                  char (&value)[100]) override {
-    uint32_t value_base =
+    assert(!rand_column_families.empty());
+    assert(!rand_keys.empty());
+
+    const std::string key_suffix = Key(rand_keys[0]);
+
+    const uint32_t value_base =
         thread->rand.Next() % thread->shared->UNKNOWN_SENTINEL;
-    size_t sz = GenerateValue(value_base, value, sizeof(value));
-    Slice v(value, sz);
-    std::string keys[10] = {"9", "8", "7", "6", "5", "4", "3", "2", "1", "0"};
-    std::string values[10] = {"9", "8", "7", "6", "5", "4", "3", "2", "1", "0"};
-    Slice value_slices[10];
+    const size_t sz = GenerateValue(value_base, value, sizeof(value));
+    const std::string value_suffix = Slice(value, sz).ToString();
+
     WriteBatch batch(0 /* reserved_bytes */, 0 /* max_bytes */,
                      FLAGS_batch_protection_bytes_per_key,
                      FLAGS_user_timestamp_size);
-    Status s;
-    auto cfh = column_families_[rand_column_families[0]];
-    std::string key_str = Key(rand_keys[0]);
-    for (int i = 0; i < 10; i++) {
-      keys[i] += key_str;
-      values[i] += v.ToString();
-      value_slices[i] = values[i];
+
+    ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]];
+    assert(cfh);
+
+    for (int i = 9; i >= 0; --i) {
+      const std::string prefix = std::to_string(i);
+
+      const std::string k = prefix + key_suffix;
+      const std::string v = prefix + value_suffix;
+
       if (FLAGS_use_merge) {
-        batch.Merge(cfh, keys[i], value_slices[i]);
+        batch.Merge(cfh, k, v);
+      } else if (FLAGS_use_put_entity_one_in > 0 &&
+                 (value_base % FLAGS_use_put_entity_one_in) == 0) {
+        batch.PutEntity(cfh, k, GenerateWideColumns(value_base, v));
       } else {
-        batch.Put(cfh, keys[i], value_slices[i]);
+        batch.Put(cfh, k, v);
       }
     }
 
-    s = db_->Write(write_opts, &batch);
+    const Status s = db_->Write(write_opts, &batch);
+
     if (!s.ok()) {
       fprintf(stderr, "multiput error: %s\n", s.ToString().c_str());
       thread->stats.AddErrors(1);
@@ -254,88 +264,122 @@ class BatchedOpsStressTest : public StressTest {
   Status TestPrefixScan(ThreadState* thread, const ReadOptions& readoptions,
                         const std::vector<int>& rand_column_families,
                         const std::vector<int64_t>& rand_keys) override {
-    size_t prefix_to_use =
-        (FLAGS_prefix_size < 0) ? 7 : static_cast<size_t>(FLAGS_prefix_size);
-    std::string key_str = Key(rand_keys[0]);
-    Slice key = key_str;
-    auto cfh = column_families_[rand_column_families[0]];
-    std::string prefixes[10] = {"0", "1", "2", "3", "4",
-                                "5", "6", "7", "8", "9"};
-    Slice prefix_slices[10];
-    ReadOptions readoptionscopy[10];
-    const Snapshot* snapshot = db_->GetSnapshot();
-    Iterator* iters[10];
-    std::string upper_bounds[10];
-    Slice ub_slices[10];
-    Status s = Status::OK();
-    for (int i = 0; i < 10; i++) {
-      prefixes[i] += key.ToString();
-      prefixes[i].resize(prefix_to_use);
-      prefix_slices[i] = Slice(prefixes[i]);
-      readoptionscopy[i] = readoptions;
-      readoptionscopy[i].snapshot = snapshot;
+    assert(!rand_column_families.empty());
+    assert(!rand_keys.empty());
+
+    const std::string key = Key(rand_keys[0]);
+
+    assert(FLAGS_prefix_size > 0);
+    const size_t prefix_to_use = static_cast<size_t>(FLAGS_prefix_size);
+
+    constexpr size_t num_prefixes = 10;
+
+    std::array<std::string, num_prefixes> prefixes;
+    std::array<Slice, num_prefixes> prefix_slices;
+    std::array<ReadOptions, num_prefixes> ro_copies;
+    std::array<std::string, num_prefixes> upper_bounds;
+    std::array<Slice, num_prefixes> ub_slices;
+    std::array<std::unique_ptr<Iterator>, num_prefixes> iters;
+
+    const Snapshot* const snapshot = db_->GetSnapshot();
+
+    ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]];
+    assert(cfh);
+
+    for (size_t i = 0; i < num_prefixes; ++i) {
+      prefixes[i] = std::to_string(i) + key;
+      prefix_slices[i] = Slice(prefixes[i].data(), prefix_to_use);
+
+      ro_copies[i] = readoptions;
+      ro_copies[i].snapshot = snapshot;
       if (thread->rand.OneIn(2) &&
           GetNextPrefix(prefix_slices[i], &(upper_bounds[i]))) {
         // For half of the time, set the upper bound to the next prefix
-        ub_slices[i] = Slice(upper_bounds[i]);
-        readoptionscopy[i].iterate_upper_bound = &(ub_slices[i]);
+        ub_slices[i] = upper_bounds[i];
+        ro_copies[i].iterate_upper_bound = &(ub_slices[i]);
       }
-      iters[i] = db_->NewIterator(readoptionscopy[i], cfh);
+
+      iters[i].reset(db_->NewIterator(ro_copies[i], cfh));
       iters[i]->Seek(prefix_slices[i]);
     }
 
-    long count = 0;
+    uint64_t count = 0;
+
     while (iters[0]->Valid() && iters[0]->key().starts_with(prefix_slices[0])) {
-      count++;
-      std::string values[10];
+      ++count;
+
+      std::array<std::string, num_prefixes> values;
+
       // get list of all values for this iteration
-      for (int i = 0; i < 10; i++) {
+      for (size_t i = 0; i < num_prefixes; ++i) {
         // no iterator should finish before the first one
         assert(iters[i]->Valid() &&
                iters[i]->key().starts_with(prefix_slices[i]));
         values[i] = iters[i]->value().ToString();
 
-        char expected_first = (prefixes[i])[0];
-        char actual_first = (values[i])[0];
+        // make sure the first character of the value is the expected digit
+        const char expected_first = prefixes[i][0];
+        const char actual_first = values[i][0];
 
         if (actual_first != expected_first) {
           fprintf(stderr, "error expected first = %c actual = %c\n",
                   expected_first, actual_first);
         }
-        (values[i])[0] = ' ';  // blank out the differing character
-      }
-      // make sure all values are equivalent
-      for (int i = 0; i < 10; i++) {
+
+        values[i][0] = ' ';  // blank out the differing character
+
+        // make sure all values are equivalent
         if (values[i] != values[0]) {
           fprintf(stderr,
-                  "error : %d, inconsistent values for prefix %s: %s, %s\n", i,
-                  prefixes[i].c_str(), StringToHex(values[0]).c_str(),
+                  "error : %" ROCKSDB_PRIszt
+                  ", inconsistent values for prefix %s: %s, %s\n",
+                  i, prefix_slices[i].ToString(/* hex */ true).c_str(),
+                  StringToHex(values[0]).c_str(),
                   StringToHex(values[i]).c_str());
           // we continue after error rather than exiting so that we can
           // find more errors if any
         }
+
+        // make sure value() and columns() are consistent
+        // note: in these tests, value base is stored after a single-digit
+        // prefix
+        Slice value_base_slice = iters[i]->value();
+        value_base_slice.remove_prefix(1);
+
+        const WideColumns expected_columns = GenerateExpectedWideColumns(
+            GetValueBase(value_base_slice), iters[i]->value());
+        if (iters[i]->columns() != expected_columns) {
+          fprintf(stderr,
+                  "error : %" ROCKSDB_PRIszt
+                  ", value and columns inconsistent for prefix %s: %s\n",
+                  i, prefix_slices[i].ToString(/* hex */ true).c_str(),
+                  DebugString(iters[i]->value(), iters[i]->columns(),
+                              expected_columns)
+                      .c_str());
+        }
+
         iters[i]->Next();
       }
     }
 
     // cleanup iterators and snapshot
-    for (int i = 0; i < 10; i++) {
+    for (size_t i = 0; i < num_prefixes; ++i) {
       // if the first iterator finished, they should have all finished
       assert(!iters[i]->Valid() ||
              !iters[i]->key().starts_with(prefix_slices[i]));
       assert(iters[i]->status().ok());
-      delete iters[i];
     }
+
     db_->ReleaseSnapshot(snapshot);
 
-    if (s.ok()) {
-      thread->stats.AddPrefixes(1, count);
-    } else {
-      fprintf(stderr, "TestPrefixScan error: %s\n", s.ToString().c_str());
-      thread->stats.AddErrors(1);
-    }
+    thread->stats.AddPrefixes(1, count);
 
-    return s;
+    return Status::OK();
+  }
+
+  Slice GetValueBaseSlice(Slice slice) override {
+    slice.remove_prefix(1);
+    return slice;
   }
 
   void VerifyDb(ThreadState* /* thread */) const override {}