rrudb 0.0.2

data/ext/rudb/NuDB/include/nudb/recover.hpp

@@ -0,0 +1,73 @@
+//
+// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef NUDB_RECOVER_HPP
+#define NUDB_RECOVER_HPP
+
+#include <nudb/error.hpp>
+#include <nudb/native_file.hpp>
+
+namespace nudb {
+
+/** Perform recovery on a database.
+
+    This implements the recovery algorithm by rolling back
+    any partially committed data. If no log file is present,
+    the function does nothing.
+
+    During the commit phase of a NuDB database, a log file
+    is generated with information that may be used to roll
+    back the results of a partial commit. This function
+    checks for the presence of a log file. If present, the
+    log file is replayed on the key and data files belonging
+    to the database, restoring the database to its state
+    before the partial commit. When @ref recover is
+    successful, it erases the log file.
+
+    It is normally not necessary to call this function
+    directly, it is called automatically when a database is
+    opened in a call to @ref basic_store::open. Callers may
+    use this function to implement auxiliary tools for
+    manipulating the database.
+
+    @par Template Parameters
+
+    @tparam Hasher The hash function to use. This type must
+    meet the requirements of @b Hasher. The hash function
+    must be the same as that used to create the database, or
+    else an error is returned.
+
+    @tparam File The type of file to use. Use the default of
+    @ref native_file unless customizing the file behavior.
+
+    @param dat_path The path to the data file.
+
+    @param key_path The path to the key file.
+
+    @param log_path The path to the log file.
+
+    @param args Optional parameters passed to File constructors.
+
+    @param ec Set to the error, if any occurred.
+*/
+template<
+    class Hasher,
+    class File = native_file,
+    class... Args>
+void
+recover(
+    path_type const& dat_path,
+    path_type const& key_path,
+    path_type const& log_path,
+    error_code& ec,
+    Args&&... args);
+
+} // nudb
+
+#include <nudb/impl/recover.ipp>
+
+#endif

data/ext/rudb/NuDB/include/nudb/rekey.hpp

@@ -0,0 +1,110 @@
+//
+// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef NUDB_REKEY_HPP
+#define NUDB_REKEY_HPP
+
+#include <nudb/error.hpp>
+#include <nudb/file.hpp>
+#include <cstddef>
+#include <cstdint>
+
+namespace nudb {
+
+/** Create a new key file from a data file.
+
+    This algorithm rebuilds a key file for the given data file.
+    It works efficiently by iterating the data file multiple times.
+    During the iteration, a contiguous block of the key file is
+    rendered in memory, then flushed to disk when the iteration is
+    complete. The size of this memory buffer is controlled by the
+    `bufferSize` parameter, larger is better. The algorithm works
+    the fastest when `bufferSize` is large enough to hold the entire
+    key file in memory; only a single iteration of the data file
+    is needed in this case.
+
+    During the rekey, spill records may be appended to the data
+    file. If the rekey operation is abnormally terminated, this
+    would normally result in a corrupted data file. To prevent this,
+    the function creates a log file using the specified path so
+    that the database can be fixed in a subsequent call to
+    @ref recover.
+
+    @note If a log file is already present, this function will
+    fail with @ref error::log_file_exists.
+
+    @par Template Parameters
+
+    @tparam Hasher The hash function to use. This type must
+    meet the requirements of @b Hasher. The hash function
+    must be the same as that used to create the database, or
+    else an error is returned.
+
+    @tparam File The type of file to use. This type must meet
+    the requirements of @b File.
+
+    @param dat_path The path to the data file.
+
+    @param key_path The path to the key file.
+
+    @param log_path The path to the log file.
+
+    @param blockSize The size of a key file block. Larger
+    blocks hold more keys but require more I/O cycles per
+    operation. The ideal block size the largest size that
+    may be read in a single I/O cycle, and device dependent.
+    The return value of @ref block_size returns a suitable
+    value for the volume of a given path.
+
+    @param loadFactor A number between zero and one
+    representing the average bucket occupancy (number of
+    items). A value of 0.5 is perfect. Lower numbers
+    waste space, and higher numbers produce negligible
+    savings at the cost of increased I/O cycles.
+
+    @param itemCount The number of items in the data file.
+
+    @param bufferSize The number of bytes to allocate for the buffer.
+
+    @param ec Set to the error if any occurred.
+
+    @param progress A function which will be called periodically
+    as the algorithm proceeds. The equivalent signature of the
+    progress function must be:
+    @code
+    void progress(
+        std::uint64_t amount,   // Amount of work done so far
+        std::uint64_t total     // Total amount of work to do
+    );
+    @endcode
+
+    @param args Optional arguments passed to @b File constructors.
+*/
+template<
+    class Hasher,
+    class File,
+    class Progress,
+    class... Args
+>
+void
+rekey(
+    path_type const& dat_path,
+    path_type const& key_path,
+    path_type const& log_path,
+    std::size_t blockSize,
+    float loadFactor,
+    std::uint64_t itemCount,
+    std::size_t bufferSize,
+    error_code& ec,
+    Progress&& progress,
+    Args&&... args);
+
+} // nudb
+
+#include <nudb/impl/rekey.ipp>
+
+#endif

data/ext/rudb/NuDB/include/nudb/store.hpp

@@ -0,0 +1,27 @@
+//
+// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef NUDB_STORE_HPP
+#define NUDB_STORE_HPP
+
+#include <nudb/basic_store.hpp>
+#include <nudb/native_file.hpp>
+#include <nudb/xxhasher.hpp>
+
+namespace nudb {
+
+/** A key/value database.
+
+    The @b Hasher used is is @ref xxhasher, which works very
+    well for almost all cases. The @b File is @ref native_file which
+    works on Windows and POSIX platforms.
+*/
+using store = basic_store<xxhasher, native_file>;
+
+} // nudb
+
+#endif

data/ext/rudb/NuDB/include/nudb/type_traits.hpp

@@ -0,0 +1,63 @@
+//
+// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef NUDB_TYPE_TRAITS_HPP
+#define NUDB_TYPE_TRAITS_HPP
+
+#include <cstddef>
+#include <cstdint>
+
+namespace nudb {
+
+#if ! GENERATING_DOCS
+
+namespace detail {
+
+// Holds a full digest
+using nhash_t = std::uint64_t;
+
+} // detail
+
+/** Holds a bucket index or bucket count.
+
+    The maximum number of buckets in a key file is 2^32-1.
+*/
+//using nbuck_t = std::uint32_t;
+using nbuck_t = std::size_t;
+
+/** Holds a key index or count in bucket.
+
+    A bucket is limited to 2^16-1 items. The practical
+    limit is lower, since a bucket cannot be larger than
+    the block size.
+*/
+//using nkey_t = std::uint16_t;
+using nkey_t = std::size_t;
+
+/** Holds a file size or offset.
+
+    Operating system support for large files is required.
+    Practically, data files cannot exceed 2^48 since offsets
+    are stored as 48 bit unsigned values.
+*/
+using noff_t = std::uint64_t;
+
+/** Holds a block, key, or value size.
+
+    Block size is limited to 2^16
+
+    Key file blocks are limited to the block size.
+
+    Value sizes are limited to 2^31-1.
+*/
+using nsize_t = std::size_t;
+
+#endif
+
+} // nudb
+
+#endif

data/ext/rudb/NuDB/include/nudb/verify.hpp

@@ -0,0 +1,200 @@
+//
+// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef NUDB_VERIFY_HPP
+#define NUDB_VERIFY_HPP
+
+#include <nudb/file.hpp>
+#include <nudb/type_traits.hpp>
+#include <nudb/detail/bucket.hpp>
+#include <nudb/detail/bulkio.hpp>
+#include <nudb/detail/format.hpp>
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <string>
+
+namespace nudb {
+
+/// Describes database statistics calculated by @ref verify.
+struct verify_info
+{
+    /** Indicates the verify algorithm used.
+
+        @li @b 0 Normal algorithm
+        @li @b 1 Fast algorith
+    */
+    int algorithm;                      // 0 = normal, 1 = fast
+
+    /// The path to the data file
+    path_type dat_path;
+
+    /// The path to the key file
+    path_type key_path;
+
+    /// The API version used to create the database
+    std::size_t version = 0;
+
+    /// The unique identifier
+    std::uint64_t uid = 0;
+
+    /// The application-defined constant
+    std::uint64_t appnum = 0;
+
+    /// The size of each key, in bytes
+    nsize_t key_size = 0;
+
+    /// The salt used in the key file
+    std::uint64_t salt = 0;
+
+    /// The salt fingerprint
+    std::uint64_t pepper = 0;
+
+    /// The block size used in the key file
+    nsize_t block_size = 0;
+
+    /// The target load factor used in the key file
+    float load_factor = 0;
+
+    /// The maximum number of keys each bucket can hold
+    nkey_t capacity = 0;
+
+    /// The number of buckets in the key file
+    nbuck_t buckets = 0;
+
+    /// The size of a bucket in bytes
+    nsize_t bucket_size = 0;
+
+    /// The size of the key file
+    noff_t key_file_size = 0;
+
+    /// The size of the data file
+    noff_t dat_file_size = 0;
+
+    /// The number of keys found
+    std::uint64_t key_count = 0;
+
+    /// The number of values found
+    std::uint64_t value_count = 0;
+
+    /// The total number of bytes occupied by values
+    std::uint64_t value_bytes = 0;
+
+    /// The number of spill records in use
+    std::uint64_t spill_count = 0;
+
+    /// The total number of spill records
+    std::uint64_t spill_count_tot = 0;
+
+    /// The number of bytes occupied by spill records in use
+    std::uint64_t spill_bytes = 0;
+
+    /// The number of bytes occupied by all spill records
+    std::uint64_t spill_bytes_tot = 0;
+
+    /// Average number of key file reads per fetch
+    float avg_fetch = 0;
+
+    /// The fraction of the data file that is wasted
+    float waste = 0;
+
+    /// The data amplification ratio
+    float overhead = 0;
+
+    /// The measured bucket load fraction
+    float actual_load = 0;
+
+    /// A histogram of the number of buckets having N spill records
+    std::array<nbuck_t, 10> hist;
+
+    /// Default constructor
+    verify_info()
+    {
+        hist.fill(0);
+    }
+};
+
+/** Verify consistency of the key and data files.
+
+    This function opens the key and data files, and
+    performs the following checks on the contents:
+
+    @li Data file header validity
+
+    @li Key file header validity
+
+    @li Data and key file header agreements
+
+    @li Check that each value is contained in a bucket
+
+    @li Check that each bucket item reflects a value
+
+    @li Ensure no values with duplicate keys
+
+    Undefined behavior results when verifying a database
+    that still has a log file. Use @ref recover on such
+    databases first.
+
+    This function selects one of two algorithms to use, the
+    normal version, and a faster version that can take advantage
+    of a buffer of sufficient size. Depending on the value of
+    the bufferSize argument, the appropriate algorithm is chosen.
+
+    A good value of bufferSize is one that is a large fraction
+    of the key file size. For example, 20% of the size of the
+    key file. Larger is better, with the highest usable value
+    depending on the size of the key file. If presented with
+    a buffer size that is too large to be of extra use, the
+    fast algorithm will simply allocate what it needs.
+
+    @par Template Parameters
+
+    @tparam Hasher The hash function to use. This type must
+    meet the requirements of @b HashFunction. The hash function
+    must be the same as that used to create the database, or
+    else an error is returned.
+
+    @param info A structure which will be default constructed
+    inside this function, and filled in if the operation completes
+    successfully. If an error is indicated, the contents of this
+    variable are undefined.
+
+    @param dat_path The path to the data file.
+
+    @param key_path The path to the key file.
+
+    @param bufferSize The number of bytes to allocate for the buffer.
+    If this number is too small, or zero, a slower algorithm will be
+    used that does not require a buffer.
+
+    @param progress A function which will be called periodically
+    as the algorithm proceeds. The equivalent signature of the
+    progress function must be:
+    @code
+    void progress(
+        std::uint64_t amount,   // Amount of work done so far
+        std::uint64_t total     // Total amount of work to do
+    );
+    @endcode
+
+    @param ec Set to the error, if any occurred.
+*/
+template<class Hasher, class Progress>
+void
+verify(
+    verify_info& info,
+    path_type const& dat_path,
+    path_type const& key_path,
+    std::size_t bufferSize,
+    Progress&& progress,
+    error_code& ec);
+
+} // nudb
+
+#include <nudb/impl/verify.ipp>
+
+#endif

data/ext/rudb/NuDB/include/nudb/version.hpp

@@ -0,0 +1,21 @@
+//
+// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef NUDB_VERSION_HPP
+#define NUDB_VERSION_HPP
+
+// follows http://semver.org
+
+//  NUDB_VERSION % 100 is the patch level
+//  NUDB_VERSION / 100 % 1000 is the minor version
+//  NUDB_VERSION / 100000 is the major version
+//
+#define NUDB_VERSION 200000
+
+#define NUDB_VERSION_STRING "2.0.0"
+
+#endif

data/ext/rudb/NuDB/include/nudb/visit.hpp

@@ -0,0 +1,63 @@
+//
+// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef NUDB_VISIT_HPP
+#define NUDB_VISIT_HPP
+
+#include <nudb/error.hpp>
+#include <nudb/file.hpp>
+
+namespace nudb {
+
+/** Visit each key/data pair in a data file.
+
+    This function will open and iterate the contents of a
+    data file, invoking the callback for each key/value
+    pair found. Only a data file is necessary, the key
+    file may be omitted.
+
+    @param path The path to the data file.
+
+    @param callback A function which will be called with
+    each item found in the data file. The equivalent signature
+    of the callback must be:
+    @code
+    void callback(
+        void const* key,        // A pointer to the item key
+        std::size_t key_size,   // The size of the key (always the same)
+        void const* data,       // A pointer to the item data
+        std::size_t data_size,  // The size of the item data
+        error_code& ec          // Indicates an error (out parameter)
+    );    
+    @endcode
+    If the callback sets ec to an error, the visit is terminated.
+
+    @param progress A function which will be called periodically
+    as the algorithm proceeds. The equivalent signature of the
+    progress function must be:
+    @code
+    void progress(
+        std::uint64_t amount,   // Amount of work done so far
+        std::uint64_t total     // Total amount of work to do
+    );
+    @endcode
+
+    @param ec Set to the error, if any occurred.
+*/
+template<class Callback, class Progress>
+void
+visit(
+    path_type const& path,
+    Callback&& callback,
+    Progress&& progress,
+    error_code& ec);
+
+} // nudb
+
+#include <nudb/impl/visit.ipp>
+
+#endif