@nxtedition/rocksdb 8.2.0 → 8.2.2

package/deps/rocksdb/rocksdb/options/options_test.cc

@@ -101,7 +101,9 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
       {"compaction_style", "kCompactionStyleLevel"},
       {"compaction_pri", "kOldestSmallestSeqFirst"},
       {"verify_checksums_in_compaction", "false"},
-      {"compaction_options_fifo", "23"},
+      {"compaction_options_fifo",
+       "{allow_compaction=true;max_table_files_size=11002244;"
+       "file_temperature_age_thresholds={{temperature=kCold;age=12345}}}"},
       {"max_sequential_skip_in_iterations", "24"},
       {"inplace_update_support", "true"},
       {"report_bg_io_stats", "true"},
@@ -128,6 +130,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
       {"blob_file_starting_level", "1"},
       {"prepopulate_blob_cache", "kDisable"},
       {"last_level_temperature", "kWarm"},
+      {"persist_user_defined_timestamps", "true"},
   };
 
   std::unordered_map<std::string, std::string> db_options_map = {
@@ -243,7 +246,18 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
   ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
   ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst);
   ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
-            static_cast<uint64_t>(23));
+            static_cast<uint64_t>(11002244));
+  ASSERT_EQ(new_cf_opt.compaction_options_fifo.allow_compaction, true);
+  ASSERT_EQ(
+      new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds.size(),
+      1);
+  ASSERT_EQ(
+      new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds[0]
+          .temperature,
+      Temperature::kCold);
+  ASSERT_EQ(
+      new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds[0].age,
+      12345);
   ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
             static_cast<uint64_t>(24));
   ASSERT_EQ(new_cf_opt.inplace_update_support, true);
@@ -269,6 +283,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
   ASSERT_EQ(new_cf_opt.prepopulate_blob_cache, PrepopulateBlobCache::kDisable);
   ASSERT_EQ(new_cf_opt.last_level_temperature, Temperature::kWarm);
   ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm);
+  ASSERT_EQ(new_cf_opt.persist_user_defined_timestamps, true);
 
   cf_options_map["write_buffer_size"] = "hello";
   ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
@@ -2161,7 +2176,7 @@ TEST_F(OptionsTest, ConvertOptionsTest) {
   const auto table_opt = table_factory->GetOptions<BlockBasedTableOptions>();
   ASSERT_NE(table_opt, nullptr);
 
-  ASSERT_EQ(table_opt->block_cache->GetCapacity(), 8UL << 20);
+  ASSERT_EQ(table_opt->block_cache->GetCapacity(), 32UL << 20);
   ASSERT_EQ(table_opt->block_size, leveldb_opt.block_size);
   ASSERT_EQ(table_opt->block_restart_interval,
             leveldb_opt.block_restart_interval);
@@ -2293,7 +2308,9 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
       {"compaction_style", "kCompactionStyleLevel"},
       {"compaction_pri", "kOldestSmallestSeqFirst"},
       {"verify_checksums_in_compaction", "false"},
-      {"compaction_options_fifo", "23"},
+      {"compaction_options_fifo",
+       "{allow_compaction=true;max_table_files_size=11002244;"
+       "file_temperature_age_thresholds={{temperature=kCold;age=12345}}}"},
       {"max_sequential_skip_in_iterations", "24"},
       {"inplace_update_support", "true"},
       {"report_bg_io_stats", "true"},
@@ -2320,6 +2337,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
       {"blob_file_starting_level", "1"},
       {"prepopulate_blob_cache", "kDisable"},
       {"last_level_temperature", "kWarm"},
+      {"persist_user_defined_timestamps", "true"},
   };
 
   std::unordered_map<std::string, std::string> db_options_map = {
@@ -2433,7 +2451,18 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
   ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
   ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst);
   ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
-            static_cast<uint64_t>(23));
+            static_cast<uint64_t>(11002244));
+  ASSERT_EQ(new_cf_opt.compaction_options_fifo.allow_compaction, true);
+  ASSERT_EQ(
+      new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds.size(),
+      1);
+  ASSERT_EQ(
+      new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds[0]
+          .temperature,
+      Temperature::kCold);
+  ASSERT_EQ(
+      new_cf_opt.compaction_options_fifo.file_temperature_age_thresholds[0].age,
+      12345);
   ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
             static_cast<uint64_t>(24));
   ASSERT_EQ(new_cf_opt.inplace_update_support, true);
@@ -2459,6 +2488,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
   ASSERT_EQ(new_cf_opt.prepopulate_blob_cache, PrepopulateBlobCache::kDisable);
   ASSERT_EQ(new_cf_opt.last_level_temperature, Temperature::kWarm);
   ASSERT_EQ(new_cf_opt.bottommost_temperature, Temperature::kWarm);
+  ASSERT_EQ(new_cf_opt.persist_user_defined_timestamps, true);
 
   cf_options_map["write_buffer_size"] = "hello";
   ASSERT_NOK(GetColumnFamilyOptionsFromMap(cf_config_options, base_cf_opt,
@@ -3923,6 +3953,36 @@ class OptionsSanityCheckTest : public OptionsParserTest,
   const std::string kOptionsFileName = "OPTIONS";
 };
 
+TEST_P(OptionsSanityCheckTest, MergeOperatorErrorMessage) {
+  ColumnFamilyOptions opts;
+  Random rnd(301);
+  opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
+  std::string merge_op_name = opts.merge_operator->Name();
+  ASSERT_OK(PersistCFOptions(opts));
+
+  // Test when going from merge operator -> nullptr
+  opts.merge_operator = nullptr;
+  Status s =
+      SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelLooselyCompatible);
+  ASSERT_TRUE(s.IsInvalidArgument());
+  std::string err_msg = s.ToString();
+  std::string specified = "The specified one is " + kNullptrString;
+  std::string persisted = "the persisted one is " + merge_op_name;
+  ASSERT_TRUE(err_msg.find(specified) != std::string::npos);
+  ASSERT_TRUE(err_msg.find(persisted) != std::string::npos);
+
+  // Test when using a different merge operator
+  opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
+  s = SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelLooselyCompatible);
+  ASSERT_TRUE(s.IsInvalidArgument());
+  err_msg = s.ToString();
+  specified =
+      "The specified one is " + std::string(opts.merge_operator->Name());
+  persisted = "the persisted one is " + merge_op_name;
+  ASSERT_TRUE(err_msg.find(specified) != std::string::npos);
+  ASSERT_TRUE(err_msg.find(persisted) != std::string::npos);
+}
+
 TEST_P(OptionsSanityCheckTest, CFOptionsSanityCheck) {
   ColumnFamilyOptions opts;
   Random rnd(301);
@@ -4068,6 +4128,30 @@ TEST_P(OptionsSanityCheckTest, CFOptionsSanityCheck) {
       SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options);
     }
   }
+
+  // persist_user_defined_timestamps
+  {
+    // Test change from true to false not allowed in loose and exact mode.
+    opts.persist_user_defined_timestamps = false;
+    ASSERT_NOK(SanityCheckCFOptions(
+        opts, ConfigOptions::kSanityLevelLooselyCompatible));
+    ASSERT_NOK(
+        SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+
+    // persist the change
+    ASSERT_OK(PersistCFOptions(opts));
+    SanityCheckCFOptions(opts, config_options_.ignore_unsupported_options);
+
+    // Test change from false to true not allowed in loose and exact mode.
+    opts.persist_user_defined_timestamps = true;
+    ASSERT_NOK(SanityCheckCFOptions(
+        opts, ConfigOptions::kSanityLevelLooselyCompatible));
+    ASSERT_NOK(
+        SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+
+    // persist the change
+    ASSERT_OK(PersistCFOptions(opts));
+  }
 }
 
 TEST_P(OptionsSanityCheckTest, DBOptionsSanityCheck) {

package/deps/rocksdb/rocksdb/port/lang.h

@@ -68,3 +68,30 @@ constexpr bool kMustFreeHeapAllocations = false;
 #else
 #define TSAN_SUPPRESSION
 #endif  // TSAN_SUPPRESSION
+
+// Compile-time CPU feature testing compatibility
+//
+// A way to be extra sure these defines have been included.
+#define ASSERT_FEATURE_COMPAT_HEADER() /* empty */
+
+// MSVC doesn't support the same defines that gcc and clang provide
+// but does some like __AVX__. Here we can infer some features from others.
+#ifdef __AVX__
+#define __SSE4_2__ 1
+#define __PCLMUL__ 1
+#endif  // __AVX__
+
+// A way to disable PCLMUL
+#ifdef NO_PCLMUL
+#undef __PCLMUL__
+#endif
+
+// popcnt is generally implied by SSE4.2
+#if defined(__SSE4_2__)
+#define __POPCNT__ 1
+#endif
+
+// A way to disable POPCNT
+#ifdef NO_POPCNT
+#undef __POPCNT__
+#endif

package/deps/rocksdb/rocksdb/port/stack_trace.cc

@@ -128,6 +128,14 @@ void PrintStackTraceLine(const char* symbol, void* frame) {
 
 #endif
 
+const char* GetLldbScriptSelectThread(long long tid) {
+  // NOTE: called from a signal handler, so no heap allocation
+  static char script[80];
+  snprintf(script, sizeof(script),
+           "script -l python -- lldb.process.SetSelectedThreadByID(%lld)", tid);
+  return script;
+}
+
 }  // namespace
 
 void PrintStack(void* frames[], int num_frames) {
@@ -152,9 +160,13 @@ void PrintStack(int first_frames_to_skip) {
   // * It doesn't appear easy to detect when ASLR is in use.
   // * With DEBUG_LEVEL < 2, backtrace() can skip frames that are not skipped
   //   in GDB.
+  //
+  // LLDB also available as an option
+  bool lldb_stack_trace = getenv("ROCKSDB_LLDB_STACK") != nullptr;
 #if defined(OS_LINUX)
   // Default true, override with ROCKSDB_BACKTRACE_STACK=1
-  bool gdb_stack_trace = getenv("ROCKSDB_BACKTRACE_STACK") == nullptr;
+  bool gdb_stack_trace =
+      !lldb_stack_trace && getenv("ROCKSDB_BACKTRACE_STACK") == nullptr;
 #else
   // Default false, override with ROCKSDB_GDB_STACK=1
   bool gdb_stack_trace = getenv("ROCKSDB_GDB_STACK") != nullptr;
@@ -164,53 +176,84 @@ void PrintStack(int first_frames_to_skip) {
   char* debug_env = getenv("ROCKSDB_DEBUG");
   bool debug = debug_env != nullptr && strlen(debug_env) > 0;
 
-  if (gdb_stack_trace || debug) {
+  if (lldb_stack_trace || gdb_stack_trace || debug) {
     // Allow ouside debugger to attach, even with Yama security restrictions
 #ifdef PR_SET_PTRACER_ANY
     (void)prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0);
 #endif
     // Try to invoke GDB, either for stack trace or debugging.
-    long long attach_id = getpid();
+    long long attach_pid = getpid();
+    // NOTE: we're in a signal handler, so no heap allocation
+    char attach_pid_str[20];
+    snprintf(attach_pid_str, sizeof(attach_pid_str), "%lld", attach_pid);
 
     // `gdb -p PID` seems to always attach to main thread, but `gdb -p TID`
     // seems to be able to attach to a particular thread in a process, which
     // makes sense as the main thread TID == PID of the process.
     // But I haven't found that gdb capability documented anywhere, so leave
     // a back door to attach to main thread.
+    long long gdb_attach_id = attach_pid;
+    // Save current thread id before fork
+    long long attach_tid = 0;
 #ifdef OS_LINUX
+    attach_tid = gettid();
     if (getenv("ROCKSDB_DEBUG_USE_PID") == nullptr) {
-      attach_id = gettid();
+      gdb_attach_id = attach_tid;
     }
 #endif
-    char attach_id_str[20];
-    snprintf(attach_id_str, sizeof(attach_id_str), "%lld", attach_id);
+
+    char gdb_attach_id_str[20];
+    snprintf(gdb_attach_id_str, sizeof(gdb_attach_id_str), "%lld",
+             gdb_attach_id);
+
     pid_t child_pid = fork();
     if (child_pid == 0) {
       // child process
       if (debug) {
-        fprintf(stderr, "Invoking GDB for debugging (ROCKSDB_DEBUG=%s)...\n",
-                debug_env);
-        execlp(/*cmd in PATH*/ "gdb", /*arg0*/ "gdb", "-p", attach_id_str,
-               (char*)nullptr);
-        return;
+        if (strcmp(debug_env, "lldb") == 0) {
+          fprintf(stderr, "Invoking LLDB for debugging (ROCKSDB_DEBUG=%s)...\n",
+                  debug_env);
+          execlp(/*cmd in PATH*/ "lldb", /*arg0*/ "lldb", "-p", attach_pid_str,
+                 /*"-Q",*/ "-o", GetLldbScriptSelectThread(attach_tid),
+                 (char*)nullptr);
+          return;
+        } else {
+          fprintf(stderr, "Invoking GDB for debugging (ROCKSDB_DEBUG=%s)...\n",
+                  debug_env);
+          execlp(/*cmd in PATH*/ "gdb", /*arg0*/ "gdb", "-p", gdb_attach_id_str,
+                 (char*)nullptr);
+          return;
+        }
       } else {
-        fprintf(stderr, "Invoking GDB for stack trace...\n");
-
-        // Skip top ~4 frames here in PrintStack
-        // See https://stackoverflow.com/q/40991943/454544
-        auto bt_in_gdb =
-            "frame apply level 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 "
-            "21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 "
-            "42 43 44 -q frame";
         // Redirect child stdout to original stderr
         dup2(2, 1);
         // No child stdin (don't use pager)
         close(0);
-        // -n : Loading config files can apparently cause failures with the
-        // other options here.
-        // -batch : non-interactive; suppress banners as much as possible
-        execlp(/*cmd in PATH*/ "gdb", /*arg0*/ "gdb", "-n", "-batch", "-p",
-               attach_id_str, "-ex", bt_in_gdb, (char*)nullptr);
+        if (lldb_stack_trace) {
+          fprintf(stderr, "Invoking LLDB for stack trace...\n");
+
+          // Skip top ~8 frames here in PrintStack
+          auto bt_in_lldb =
+              "script -l python -- for f in lldb.thread.frames[8:]: print(f)";
+          execlp(/*cmd in PATH*/ "lldb", /*arg0*/ "lldb", "-p", attach_pid_str,
+                 "-b", "-Q", "-o", GetLldbScriptSelectThread(attach_tid), "-o",
+                 bt_in_lldb, (char*)nullptr);
+        } else {
+          // gdb_stack_trace
+          fprintf(stderr, "Invoking GDB for stack trace...\n");
+
+          // Skip top ~4 frames here in PrintStack
+          // See https://stackoverflow.com/q/40991943/454544
+          auto bt_in_gdb =
+              "frame apply level 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 "
+              "21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 "
+              "42 43 44 -q frame";
+          // -n : Loading config files can apparently cause failures with the
+          // other options here.
+          // -batch : non-interactive; suppress banners as much as possible
+          execlp(/*cmd in PATH*/ "gdb", /*arg0*/ "gdb", "-n", "-batch", "-p",
+                 gdb_attach_id_str, "-ex", bt_in_gdb, (char*)nullptr);
+        }
         return;
       }
     } else {

package/deps/rocksdb/rocksdb/src.mk

@@ -377,6 +377,7 @@ STRESS_LIB_SOURCES =                                            \
   db_stress_tool/db_stress_test_base.cc                        \
   db_stress_tool/db_stress_tool.cc                             \
   db_stress_tool/expected_state.cc                             \
+  db_stress_tool/expected_value.cc                             \
   db_stress_tool/no_batched_ops_stress.cc                      \
   db_stress_tool/multi_ops_txns_stress.cc                      \
 
@@ -459,6 +460,7 @@ TEST_MAIN_SOURCES =                                                     \
   db/db_bloom_filter_test.cc                                            \
   db/db_compaction_filter_test.cc                                       \
   db/db_compaction_test.cc                                              \
+  db/db_clip_test.cc                                                    \
   db/db_dynamic_level_test.cc                                           \
   db/db_encryption_test.cc                                              \
   db/db_flush_test.cc                                                   \

package/deps/rocksdb/rocksdb/table/block_based/binary_search_index_reader.cc

@@ -46,9 +46,8 @@ InternalIteratorBase<IndexValue>* BinarySearchIndexReader::NewIterator(
   const BlockBasedTable::Rep* rep = table()->get_rep();
   const bool no_io = (read_options.read_tier == kBlockCacheTier);
   CachableEntry<Block> index_block;
-  const Status s =
-      GetOrReadIndexBlock(no_io, read_options.rate_limiter_priority,
-                          get_context, lookup_context, &index_block);
+  const Status s = GetOrReadIndexBlock(no_io, get_context, lookup_context,
+                                       &index_block, read_options);
   if (!s.ok()) {
     if (iter != nullptr) {
       iter->Invalidate(s);