@nxtedition/rocksdb 5.2.40 → 6.0.1

package/BUILDING.md

@@ -0,0 +1,10 @@
+# LINUX
+
+- Run `./configure` in `deps/liburing`.
+- Install and build zstd with `CFLAGS="-O3 -fPIC" make -C lib libzstd.a` and copy to `/usr/lib/x86_64-linux-gnu/libzstd_pic.a`.
+- Put headers at `/usr/lib/x86_64-linux-gnu/include`.
+
+# OSX
+
+- Run `./configure` in `deps/liburing`.
+- `brew install zstd@1.5.2`

package/binding.cc

@@ -598,10 +598,16 @@ NAPI_METHOD(db_open) {
   const auto location = ToString(env, argv[1]);
   options.create_if_missing = BooleanProperty(env, argv[2], "createIfMissing").value_or(true);
   options.error_if_exists = BooleanProperty(env, argv[2], "errorIfExists").value_or(false);
-  options.compression = BooleanProperty(env, argv[2], "compression").value_or((true)) ? rocksdb::kSnappyCompression
+  options.compression = BooleanProperty(env, argv[2], "compression").value_or((true)) ? rocksdb::kZSTD
                                                                                       : rocksdb::kNoCompression;
-  options.use_adaptive_mutex = true;
-  options.enable_pipelined_write = true;
+  if (options.compression == rocksdb::kZSTD) {
+    options.compression_opts.max_dict_bytes = 16 * 1024;
+    options.compression_opts.zstd_max_train_bytes = 16 * 1024 * 100;
+    // options.compression_opts.parallel_threads
+  }
+
+  options.use_adaptive_mutex = BooleanProperty(env, argv[2], "useAdaptiveMutex").value_or(true);
+  options.enable_pipelined_write = BooleanProperty(env, argv[2], "enablePipelinedWrite").value_or(true);
   options.max_background_jobs =
       Uint32Property(env, argv[2], "maxBackgroundJobs").value_or(std::thread::hardware_concurrency() / 4);
 
@@ -911,46 +917,72 @@ NAPI_METHOD(db_clear) {
   const auto lte = StringProperty(env, argv[1], "lte");
   const auto gt = StringProperty(env, argv[1], "gt");
   const auto gte = StringProperty(env, argv[1], "gte");
+  
+  if (!reverse && limit == -1 && (lte || lt)) {
+    rocksdb::Slice begin;
+    if (gte) {
+      begin = *gte;
+    } else if (gt) {
+      begin = *gt + '\0';
+    }
 
-  // TODO (perf): Use DeleteRange.
+    rocksdb::Slice end;
+    if (lte) {
+      end = *lte + '\0';
+    } else if (lt) {
+      end = *lt;
+    } else {
+      assert(false);
+    }
 
-  BaseIterator it(database, reverse, lt, lte, gt, gte, limit, false);
+    if (begin.compare(end) > 0) {
+      return ToError(env, rocksdb::Status::OK());
+    }
 
-  it.SeekToRange();
+    rocksdb::WriteOptions options;
+    const auto status = database->db_->DeleteRange(options, database->db_->DefaultColumnFamily(), begin, end);
+    return ToError(env, status);
+  } else {
+    // TODO (perf): Use DeleteRange.
 
-  // TODO: add option
-  const uint32_t hwm = 16 * 1024;
+    BaseIterator it(database, reverse, lt, lte, gt, gte, limit, false);
 
-  rocksdb::WriteBatch batch;
-  rocksdb::WriteOptions options;
-  rocksdb::Status status;
+    it.SeekToRange();
 
-  while (true) {
-    size_t bytesRead = 0;
+    // TODO: add option
+    const uint32_t hwm = 16 * 1024;
 
-    while (bytesRead <= hwm && it.Valid() && it.Increment()) {
-      const auto key = it.CurrentKey();
-      batch.Delete(key);
-      bytesRead += key.size();
-      it.Next();
-    }
+    rocksdb::WriteBatch batch;
+    rocksdb::WriteOptions options;
+    rocksdb::Status status;
 
-    status = it.Status();
-    if (!status.ok() || bytesRead == 0) {
-      break;
-    }
+    while (true) {
+      size_t bytesRead = 0;
 
-    status = database->db_->Write(options, &batch);
-    if (!status.ok()) {
-      break;
-    }
+      while (bytesRead <= hwm && it.Valid() && it.Increment()) {
+        const auto key = it.CurrentKey();
+        batch.Delete(key);
+        bytesRead += key.size();
+        it.Next();
+      }
 
-    batch.Clear();
-  }
+      status = it.Status();
+      if (!status.ok() || bytesRead == 0) {
+        break;
+      }
 
-  it.Close();
+      status = database->db_->Write(options, &batch);
+      if (!status.ok()) {
+        break;
+      }
 
-  return ToError(env, status);
+      batch.Clear();
+    }
+
+    it.Close();
+
+    return ToError(env, status);
+  }
 }
 
 NAPI_METHOD(db_get_property) {

package/binding.gyp

@@ -61,7 +61,7 @@
                 "-Wno-ignored-qualifiers"
               ],
               "OTHER_CPLUSPLUSFLAGS": [
-                "-mmacosx-version-min=10.15",
+                "-mmacosx-version-min=12.2.1",
                 "-std=c++20",
                 "-fno-omit-frame-pointer",
                 "-momit-leaf-frame-pointer",
@@ -70,7 +70,7 @@
               ],
               "GCC_ENABLE_CPP_RTTI": "YES",
               "GCC_ENABLE_CPP_EXCEPTIONS": "YES",
-              "MACOSX_DEPLOYMENT_TARGET": "10.15"
+              "MACOSX_DEPLOYMENT_TARGET": "12.2.1"
             }
           }
         ]

package/deps/rocksdb/rocksdb.gyp

@@ -5,16 +5,23 @@
       "target_name": "rocksdb",
       "type": "static_library",
       "standalone_static_library": 1,
-      "dependencies": ["../snappy/snappy.gyp:snappy"],
-      "direct_dependent_settings": { "include_dirs": ["rocksdb/include/"] },
+      "direct_dependent_settings": { 
+        "include_dirs": ["rocksdb/include/"] 
+      },
       "defines": [
-        "SNAPPY=1",
+        "ZSTD=1",
+        "ZSTD_STATIC_LINKING_ONLY=1",
+        # "ROCKSDB_JEMALLOC=1",
+        # "JEMALLOC_NO_DEMANGLE=1",
         "ROCKSDB_BACKTRACE=1",
         "ROCKSDB_SUPPORT_THREAD_LOCAL=1",
         "NIOSTATS_CONTEXT=1",
         "NPERF_CONTEXT=1"
       ],
-      "include_dirs": ["rocksdb/", "rocksdb/include/"],
+      "include_dirs": [
+        "rocksdb/", 
+        "rocksdb/include/"
+      ],
       "conditions": [
         [
           "OS == 'win'",
@@ -84,8 +91,6 @@
               "ROCKSDB_RANGESYNC_PRESENT=1",
               "ROCKSDB_SCHED_GETCPU_PRESENT=1",
               "ROCKSDB_IOURING_PRESENT=1",
-              # "ROCKSDB_JEMALLOC=1",
-              # "JEMALLOC_NO_DEMANGLE=1",
               "HAVE_SSE42=1",
               "HAVE_BMI=1",
               "HAVE_LZCNT=1",
@@ -97,7 +102,19 @@
               # "NUMA=1",
               # "TBB=1",
             ],
-            "dependencies": ["../liburing/liburing.gyp:liburing"],
+            "direct_dependent_settings": {
+              "libraries": [
+                "/usr/lib/x86_64-linux-gnu/libzstd_pic.a",
+                # "/usr/lib/x86_64-linux-gnu/libjemalloc_pic.a",
+              ],
+            },
+            "include_dirs": [
+              "/usr/lib/x86_64-linux-gnu/include",
+              # "/opt/homebrew/Cellar/jemalloc/5.3.0/include"
+            ],
+            "dependencies": [
+              "../liburing/liburing.gyp:liburing"
+            ],
             "cflags": [
               "-msse4.2",
               "-mpclmul",
@@ -109,16 +126,26 @@
             "ccflags": ["-flto"],
             "cflags!": ["-fno-exceptions"],
             "cflags_cc!": ["-fno-exceptions"],
-            "ldflags": ["-flto", "-fuse-linker-plugin"]
-          }
+            "ldflags": ["-flto", "-fuse-linker-plugin"],
+          },
         ],
         [
           "OS == 'mac'",
           {
             "defines": ["OS_MACOSX=1"],
+            "direct_dependent_settings": {
+              "libraries": [
+                "/opt/homebrew/Cellar/zstd/1.5.2/lib/libzstd.a",
+                # "/opt/homebrew/Cellar/jemalloc/5.3.0/lib/libjemalloc.a"
+              ],
+            },
+            "include_dirs": [
+              "/opt/homebrew/Cellar/zstd/1.5.2/include",
+              # "/opt/homebrew/Cellar/jemalloc/5.3.0/include"
+            ],
             "xcode_settings": {
               "OTHER_CPLUSPLUSFLAGS": [
-                "-mmacosx-version-min=10.15",
+                "-mmacosx-version-min=12.2.1",
                 "-std=c++20",
                 "-fno-omit-frame-pointer",
                 "-momit-leaf-frame-pointer",
@@ -127,8 +154,8 @@
               ],
               "GCC_ENABLE_CPP_RTTI": "YES",
               "GCC_ENABLE_CPP_EXCEPTIONS": "YES",
-              "MACOSX_DEPLOYMENT_TARGET": "10.15"
-            }
+              "MACOSX_DEPLOYMENT_TARGET": "12.2.1"
+            },
           }
         ]
       ],
@@ -362,12 +389,12 @@
         "rocksdb/util/xxhash.cc",
         "rocksdb/utilities/agg_merge/agg_merge.cc",
         "rocksdb/utilities/backup/backup_engine.cc",
-        # "rocksdb/utilities/blob_db/blob_compaction_filter.cc",
-        # "rocksdb/utilities/blob_db/blob_db.cc",
-        # "rocksdb/utilities/blob_db/blob_db_impl.cc",
-        # "rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc",
-        # "rocksdb/utilities/blob_db/blob_dump_tool.cc",
-        # "rocksdb/utilities/blob_db/blob_file.cc",
+        "rocksdb/utilities/blob_db/blob_compaction_filter.cc",
+        "rocksdb/utilities/blob_db/blob_db.cc",
+        "rocksdb/utilities/blob_db/blob_db_impl.cc",
+        "rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc",
+        "rocksdb/utilities/blob_db/blob_dump_tool.cc",
+        "rocksdb/utilities/blob_db/blob_file.cc",
         "rocksdb/utilities/cache_dump_load.cc",
         "rocksdb/utilities/cache_dump_load_impl.cc",
         "rocksdb/utilities/checkpoint/checkpoint_impl.cc",
@@ -375,11 +402,11 @@
         "rocksdb/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
         "rocksdb/utilities/counted_fs.cc",
         "rocksdb/utilities/debug.cc",
-        # "rocksdb/utilities/env_mirror.cc",
-        # "rocksdb/utilities/env_timed.cc",
-        # "rocksdb/utilities/fault_injection_env.cc",
-        # "rocksdb/utilities/fault_injection_fs.cc",
-        # "rocksdb/utilities/fault_injection_secondary_cache.cc",
+        "rocksdb/utilities/env_mirror.cc",
+        "rocksdb/utilities/env_timed.cc",
+        "rocksdb/utilities/fault_injection_env.cc",
+        "rocksdb/utilities/fault_injection_fs.cc",
+        "rocksdb/utilities/fault_injection_secondary_cache.cc",
         "rocksdb/utilities/leveldb_options/leveldb_options.cc",
         "rocksdb/utilities/memory/memory_util.cc",
         "rocksdb/utilities/merge_operators.cc",
@@ -393,13 +420,13 @@
         "rocksdb/utilities/object_registry.cc",
         "rocksdb/utilities/option_change_migration/option_change_migration.cc",
         "rocksdb/utilities/options/options_util.cc",
-        # "rocksdb/utilities/persistent_cache/block_cache_tier.cc",
-        # "rocksdb/utilities/persistent_cache/block_cache_tier_file.cc",
-        # "rocksdb/utilities/persistent_cache/block_cache_tier_metadata.cc",
-        # "rocksdb/utilities/persistent_cache/persistent_cache_tier.cc",
-        # "rocksdb/utilities/persistent_cache/volatile_tier_impl.cc",
-        # "rocksdb/utilities/simulator_cache/cache_simulator.cc",
-        # "rocksdb/utilities/simulator_cache/sim_cache.cc",
+        "rocksdb/utilities/persistent_cache/block_cache_tier.cc",
+        "rocksdb/utilities/persistent_cache/block_cache_tier_file.cc",
+        "rocksdb/utilities/persistent_cache/block_cache_tier_metadata.cc",
+        "rocksdb/utilities/persistent_cache/persistent_cache_tier.cc",
+        "rocksdb/utilities/persistent_cache/volatile_tier_impl.cc",
+        "rocksdb/utilities/simulator_cache/cache_simulator.cc",
+        "rocksdb/utilities/simulator_cache/sim_cache.cc",
         "rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.cc",
         "rocksdb/utilities/trace/file_trace_reader_writer.cc",
         "rocksdb/utilities/trace/replayer_impl.cc",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nxtedition/rocksdb",
-  "version": "5.2.40",
+  "version": "6.0.1",
   "description": "A low-level Node.js RocksDB binding",
   "license": "MIT",
   "main": "index.js",

package/deps/snappy/freebsd/config.h

@@ -1,135 +0,0 @@
-/* config.h.  Generated from config.h.in by configure.  */
-/* config.h.in.  Generated from configure.ac by autoheader.  */
-
-/* Define if building universal (internal helper macro) */
-/* #undef AC_APPLE_UNIVERSAL_BUILD */
-
-/* Define to 1 if the compiler supports __builtin_ctz and friends. */
-#define HAVE_BUILTIN_CTZ 1
-
-/* Define to 1 if the compiler supports __builtin_expect. */
-#define HAVE_BUILTIN_EXPECT 1
-
-/* Define to 1 if you have the <byteswap.h> header file. */
-/* #undef HAVE_BYTESWAP_H */
-
-/* Define to 1 if you have the <dlfcn.h> header file. */
-#define HAVE_DLFCN_H 1
-
-/* Use the gflags package for command-line parsing. */
-/* #undef HAVE_GFLAGS */
-
-/* Defined when Google Test is available. */
-/* #undef HAVE_GTEST */
-
-/* Define to 1 if you have the <inttypes.h> header file. */
-#define HAVE_INTTYPES_H 1
-
-/* Define to 1 if you have the `fastlz' library (-lfastlz). */
-/* #undef HAVE_LIBFASTLZ */
-
-/* Define to 1 if you have the `lzf' library (-llzf). */
-/* #undef HAVE_LIBLZF */
-
-/* Define to 1 if you have the `lzo2' library (-llzo2). */
-/* #undef HAVE_LIBLZO2 */
-
-/* Define to 1 if you have the `quicklz' library (-lquicklz). */
-/* #undef HAVE_LIBQUICKLZ */
-
-/* Define to 1 if you have the `z' library (-lz). */
-#define HAVE_LIBZ 1
-
-/* Define to 1 if you have the <memory.h> header file. */
-#define HAVE_MEMORY_H 1
-
-/* Define to 1 if you have the <stddef.h> header file. */
-#define HAVE_STDDEF_H 1
-
-/* Define to 1 if you have the <stdint.h> header file. */
-#define HAVE_STDINT_H 1
-
-/* Define to 1 if you have the <stdlib.h> header file. */
-#define HAVE_STDLIB_H 1
-
-/* Define to 1 if you have the <strings.h> header file. */
-#define HAVE_STRINGS_H 1
-
-/* Define to 1 if you have the <string.h> header file. */
-#define HAVE_STRING_H 1
-
-/* Define to 1 if you have the <sys/byteswap.h> header file. */
-/* #undef HAVE_SYS_BYTESWAP_H */
-
-/* Define to 1 if you have the <sys/endian.h> header file. */
-#define HAVE_SYS_ENDIAN_H 1
-
-/* Define to 1 if you have the <sys/mman.h> header file. */
-#define HAVE_SYS_MMAN_H 1
-
-/* Define to 1 if you have the <sys/resource.h> header file. */
-#define HAVE_SYS_RESOURCE_H 1
-
-/* Define to 1 if you have the <sys/stat.h> header file. */
-#define HAVE_SYS_STAT_H 1
-
-/* Define to 1 if you have the <sys/time.h> header file. */
-#define HAVE_SYS_TIME_H 1
-
-/* Define to 1 if you have the <sys/types.h> header file. */
-#define HAVE_SYS_TYPES_H 1
-
-/* Define to 1 if you have the <sys/uio.h> header file. */
-#define HAVE_SYS_UIO_H 1
-
-/* Define to 1 if you have the <unistd.h> header file. */
-#define HAVE_UNISTD_H 1
-
-/* Define to 1 if you have the <windows.h> header file. */
-/* #undef HAVE_WINDOWS_H */
-
-/* Define to the sub-directory in which libtool stores uninstalled libraries.
-   */
-#define LT_OBJDIR ".libs/"
-
-/* Name of package */
-#define PACKAGE "snappy"
-
-/* Define to the address where bug reports for this package should be sent. */
-#define PACKAGE_BUGREPORT ""
-
-/* Define to the full name of this package. */
-#define PACKAGE_NAME "snappy"
-
-/* Define to the full name and version of this package. */
-#define PACKAGE_STRING "snappy 1.1.4"
-
-/* Define to the one symbol short name of this package. */
-#define PACKAGE_TARNAME "snappy"
-
-/* Define to the version of this package. */
-#define PACKAGE_VERSION "1.1.4"
-
-/* Define to 1 if you have the ANSI C header files. */
-#define STDC_HEADERS 1
-
-/* Version number of package */
-#define VERSION "1.1.4"
-
-/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
-   significant byte first (like Motorola and SPARC, unlike Intel). */
-#if defined AC_APPLE_UNIVERSAL_BUILD
-# if defined __BIG_ENDIAN__
-#  define WORDS_BIGENDIAN 1
-# endif
-#else
-# ifndef WORDS_BIGENDIAN
-/* #  undef WORDS_BIGENDIAN */
-# endif
-#endif
-
-/* Define to `unsigned int' if <sys/types.h> does not define. */
-/* #undef size_t */
-
-/* Define to `int' if <sys/types.h> does not define. */
-/* #undef ssize_t */

package/deps/snappy/freebsd/snappy-stubs-public.h

@@ -1,100 +0,0 @@
-// Copyright 2011 Google Inc. All Rights Reserved.
-// Author: sesse@google.com (Steinar H. Gunderson)
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Various type stubs for the open-source version of Snappy.
-//
-// This file cannot include config.h, as it is included from snappy.h,
-// which is a public header. Instead, snappy-stubs-public.h is generated by
-// from snappy-stubs-public.h.in at configure time.
-
-#ifndef THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
-#define THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
-
-#if 1
-#include <stdint.h>
-#endif
-
-#if 1
-#include <stddef.h>
-#endif
-
-#if 1
-#include <sys/uio.h>
-#endif
-
-#define SNAPPY_MAJOR 1
-#define SNAPPY_MINOR 1
-#define SNAPPY_PATCHLEVEL 4
-#define SNAPPY_VERSION \
-    ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
-
-#include <string>
-
-namespace snappy {
-
-#if 1
-typedef int8_t int8;
-typedef uint8_t uint8;
-typedef int16_t int16;
-typedef uint16_t uint16;
-typedef int32_t int32;
-typedef uint32_t uint32;
-typedef int64_t int64;
-typedef uint64_t uint64;
-#else
-typedef signed char int8;
-typedef unsigned char uint8;
-typedef short int16;
-typedef unsigned short uint16;
-typedef int int32;
-typedef unsigned int uint32;
-typedef long long int64;
-typedef unsigned long long uint64;
-#endif
-
-typedef std::string string;
-
-#ifndef DISALLOW_COPY_AND_ASSIGN
-#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
-  TypeName(const TypeName&);               \
-  void operator=(const TypeName&)
-#endif
-
-#if !1
-// Windows does not have an iovec type, yet the concept is universally useful.
-// It is simple to define it ourselves, so we put it inside our own namespace.
-struct iovec {
-	void* iov_base;
-	size_t iov_len;
-};
-#endif
-
-}  // namespace snappy
-
-#endif  // THIRD_PARTY_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_