rrudb 0.0.2

data/ext/rudb/NuDB/include/nudb/impl/posix_file.ipp

@@ -0,0 +1,248 @@
+//
+// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef NUDB_IMPL_POSIX_FILE_IPP
+#define NUDB_IMPL_POSIX_FILE_IPP
+
+#include <boost/assert.hpp>
+#include <limits.h>
+
+namespace nudb {
+
+inline
+posix_file::
+~posix_file()
+{
+    close();
+}
+
+inline
+posix_file::
+posix_file(posix_file &&other)
+    : fd_(other.fd_)
+{
+    other.fd_ = -1;
+}
+
+inline
+posix_file&
+posix_file::
+operator=(posix_file&& other)
+{
+    if(&other == this)
+        return *this;
+    close();
+    fd_ = other.fd_;
+    other.fd_ = -1;
+    return *this;
+}
+
+inline
+void
+posix_file::
+close()
+{
+    if(fd_ != -1)
+    {
+        ::close(fd_);
+        fd_ = -1;
+    }
+}
+
+inline
+void
+posix_file::
+create(file_mode mode, path_type const& path, error_code& ec)
+{
+    auto const result = flags(mode);
+    BOOST_ASSERT(! is_open());
+    fd_ = ::open(path.c_str(), result.first | O_CREAT | O_EXCL, 0644);
+    if(fd_ == -1)
+        return last_err(ec);
+#ifndef __APPLE__
+    if(::posix_fadvise(fd_, 0, 0, result.second) != 0)
+        return last_err(ec);
+#endif
+}
+
+inline
+void
+posix_file::
+open(file_mode mode, path_type const& path, error_code& ec)
+{
+    BOOST_ASSERT(! is_open());
+    auto const result = flags(mode);
+    fd_ = ::open(path.c_str(), result.first);
+    if(fd_ == -1)
+        return last_err(ec);
+#ifndef __APPLE__
+    if(::posix_fadvise(fd_, 0, 0, result.second) != 0)
+        return last_err(ec);
+#endif
+}
+
+inline
+void
+posix_file::
+erase(path_type const& path, error_code& ec)
+{
+    if(::unlink(path.c_str()) != 0)
+    {
+        int const ev = errno;
+        return err(ev, ec);
+    }
+}
+
+inline
+std::uint64_t
+posix_file::
+size(error_code& ec) const
+{
+    static_assert(sizeof(stat::st_size) == sizeof(std::uint64_t), "");
+    struct stat st;
+    if(::fstat(fd_, &st) != 0)
+    {
+        last_err(ec);
+        return 0;
+    }
+    return st.st_size;
+}
+inline
+void
+posix_file::
+read(std::uint64_t offset,
+     void* buffer, std::size_t bytes, error_code& ec)
+{
+    static_assert(sizeof(off_t) >= sizeof(offset), "");
+    while(bytes > 0)
+    {
+        auto const amount = static_cast<ssize_t>(
+            std::min(bytes, static_cast<std::size_t>(SSIZE_MAX)));
+        auto const n = ::pread(fd_, buffer, amount, offset);
+        if(n == -1)
+        {
+            auto const ev = errno;
+            if(ev == EINTR)
+                continue;
+            return err(ev, ec);
+        }
+        if(n == 0)
+        {
+            ec = error::short_read;
+            return;
+        }
+        offset += n;
+        bytes -= n;
+        buffer = reinterpret_cast<char*>(buffer) + n;
+    }
+}
+
+inline
+void
+posix_file::
+write(std::uint64_t offset,
+      void const* buffer, std::size_t bytes, error_code& ec)
+{
+    static_assert(sizeof(off_t) >= sizeof(offset), "");
+    while(bytes > 0)
+    {
+        auto const amount = static_cast<ssize_t>(
+            std::min(bytes, static_cast<std::size_t>(SSIZE_MAX)));
+        auto const n = ::pwrite(fd_, buffer, amount, offset);
+        if(n == -1)
+        {
+            auto const ev = errno;
+            if(ev == EINTR)
+                continue;
+            return err(ev, ec);
+        }
+        offset += n;
+        bytes -= n;
+        buffer = reinterpret_cast<char const*>(buffer) + n;
+    }
+}
+
+inline
+void
+posix_file::
+sync(error_code& ec)
+{
+    for(;;)
+    {
+        if(::fsync(fd_) == 0)
+            break;
+        auto const ev = errno;
+        if(ev == EINTR)
+            continue;
+        return err(ev, ec);
+    }
+}
+
+inline
+void
+posix_file::
+trunc(std::uint64_t length, error_code& ec)
+{
+    for(;;)
+    {
+        if(::ftruncate(fd_, length) == 0)
+            break;
+        auto const ev = errno;
+        if(ev == EINTR)
+            continue;
+        return err(ev, ec);
+    }
+}
+
+inline
+std::pair<int, int>
+posix_file::
+flags(file_mode mode)
+{
+    std::pair<int, int> result;
+    switch(mode)
+    {
+    case file_mode::scan:
+        result.first =
+            O_RDONLY;
+    #ifndef __APPLE__
+        result.second =
+            POSIX_FADV_SEQUENTIAL;
+    #endif
+        break;
+    case file_mode::read:
+        result.first =
+            O_RDONLY;
+    #ifndef __APPLE__
+        result.second =
+            POSIX_FADV_RANDOM;
+    #endif
+        break;
+    case file_mode::append:
+        result.first =
+            O_RDWR |
+            O_APPEND;
+    #ifndef __APPLE__
+        result.second =
+            POSIX_FADV_RANDOM;
+    #endif
+        break;
+    case file_mode::write:
+        result.first =
+            O_RDWR;
+    #ifndef __APPLE__
+        result.second =
+            POSIX_FADV_NORMAL;
+    #endif
+        break;
+    }
+    return result;
+}
+
+} // nudb
+
+#endif

data/ext/rudb/NuDB/include/nudb/impl/recover.ipp

@@ -0,0 +1,209 @@
+//
+// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef NUDB_IMPL_RECOVER_IPP
+#define NUDB_IMPL_RECOVER_IPP
+
+#include <nudb/concepts.hpp>
+#include <nudb/file.hpp>
+#include <nudb/type_traits.hpp>
+#include <nudb/detail/bucket.hpp>
+#include <nudb/detail/bulkio.hpp>
+#include <nudb/detail/format.hpp>
+#include <boost/assert.hpp>
+#include <algorithm>
+#include <cstddef>
+#include <string>
+
+namespace nudb {
+
+template<
+    class Hasher,
+    class File,
+    class... Args>
+void
+recover(
+    path_type const& dat_path,
+    path_type const& key_path,
+    path_type const& log_path,
+    error_code& ec,
+    Args&&... args)
+{
+    static_assert(is_File<File>::value,
+        "File requirements not met");
+    static_assert(is_Hasher<Hasher>::value,
+        "Hasher requirements not met");
+    using namespace detail;
+
+    // Open data file
+    File df{args...};
+    df.open(file_mode::write, dat_path, ec);
+    if(ec)
+        return;
+    auto const dataFileSize = df.size(ec);
+    if(ec)
+        return;
+    dat_file_header dh;
+    read(df, dh, ec);
+    if(ec)
+        return;
+    verify(dh, ec);
+    if(ec)
+        return;
+
+    // Open key file
+    File kf{args...};
+    kf.open(file_mode::write, key_path, ec);
+    if(ec)
+        return;
+    auto const keyFileSize = kf.size(ec);
+    if(ec)
+        return;
+    if(keyFileSize <= key_file_header::size)
+    {
+        kf.close();
+        erase_file(log_path, ec);
+        if(ec)
+            return;
+        File::erase(key_path, ec);
+        if(ec)
+            return;
+        ec = error::no_key_file;
+        return;
+    }
+
+    // Open log file
+    File lf{args...};
+    lf.open(file_mode::append, log_path, ec);
+    if(ec == errc::no_such_file_or_directory)
+    {
+        ec = {};
+        return;
+    }
+    if(ec)
+        return;
+    auto const logFileSize = lf.size(ec);
+    if(ec)
+        return;
+    // Read log file header
+    log_file_header lh;
+    read(lf, lh, ec);
+    if(ec == error::short_read)
+    {
+        BOOST_ASSERT(keyFileSize > key_file_header::size);
+        ec = {};
+        goto clear_log;
+    }
+    if(ec)
+        return;
+    verify<Hasher>(lh, ec);
+    if(ec)
+        return;
+    if(lh.key_file_size == 0)
+        goto trunc_files;
+    {
+        // Read key file header
+        key_file_header kh;
+        read(kf, kh, ec);
+        if(ec)
+            return;
+        verify<Hasher>(kh, ec);
+        if(ec)
+            return;
+        verify<Hasher>(dh, kh, ec);
+        if(ec)
+            return;
+        verify<Hasher>(kh, lh, ec);
+        if(ec)
+            return;
+
+        auto const readSize = 1024 * kh.block_size;
+        auto const bucketSize = bucket_size(kh.capacity);
+        buffer buf{kh.block_size};
+        bucket b{kh.block_size, buf.get()};
+        bulk_reader<File> r{lf,
+            log_file_header::size, logFileSize, readSize};
+        while(! r.eof())
+        {
+            // Log Record
+            auto is = r.prepare(field<std::uint64_t>::size, ec);
+            // Log file is incomplete, so roll back.
+            if(ec == error::short_read)
+            {
+                ec = {};
+                break;
+            }
+            if(ec)
+                return;
+            nsize_t n;
+            {
+                std::uint64_t v;
+                // VFALCO This should have been a uint32_t
+                read<std::uint64_t>(is, v); // Index
+                BOOST_ASSERT(v <= std::numeric_limits<std::uint32_t>::max());
+                n = static_cast<nsize_t>(v);
+            }
+            b.read(r, ec);                  // Bucket
+            if(ec == error::short_read)
+            {
+                ec = {};
+                break;
+            }
+            if(b.spill() && b.spill() + bucketSize > dataFileSize)
+            {
+                ec = error::invalid_log_spill;
+                return;
+            }
+            if(n > kh.buckets)
+            {
+                ec = error::invalid_log_index;
+                return;
+            }
+            b.write(kf, static_cast<noff_t>(n + 1) * kh.block_size, ec);
+            if(ec)
+                return;
+        }
+    }
+trunc_files:
+    df.trunc(lh.dat_file_size, ec);
+    if(ec)
+        return;
+    df.sync(ec);
+    if(ec)
+        return;
+    if(lh.key_file_size != 0)
+    {
+        kf.trunc(lh.key_file_size, ec);
+        if(ec)
+            return;
+        kf.sync(ec);
+        if(ec)
+            return;
+    }
+    else
+    {
+        kf.close();
+        File::erase(key_path, ec);
+        if(ec)
+            return;
+    }
+clear_log:
+    lf.trunc(0, ec);
+    if(ec)
+        return;
+    lf.sync(ec);
+    if(ec)
+        return;
+    lf.close();
+    File::erase(log_path, ec);
+    if(ec)
+        return;
+}
+
+} // nudb
+
+#endif

data/ext/rudb/NuDB/include/nudb/impl/rekey.ipp

@@ -0,0 +1,248 @@
+//
+// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+
+#ifndef NUDB_IMPL_REKEY_IPP
+#define NUDB_IMPL_REKEY_IPP
+
+#include <nudb/concepts.hpp>
+#include <nudb/create.hpp>
+#include <nudb/detail/bucket.hpp>
+#include <nudb/detail/bulkio.hpp>
+#include <nudb/detail/format.hpp>
+#include <cmath>
+
+namespace nudb {
+
+// VFALCO Should this delete the key file on an error?
+template<
+    class Hasher,
+    class File,
+    class Progress,
+    class... Args
+>
+void
+rekey(
+    path_type const& dat_path,
+    path_type const& key_path,
+    path_type const& log_path,
+    std::size_t blockSize,
+    float loadFactor,
+    std::uint64_t itemCount,
+    std::size_t bufferSize,
+    error_code& ec,
+    Progress&& progress,
+    Args&&... args)
+{
+    static_assert(is_File<File>::value,
+        "File requirements not met");
+    static_assert(is_Hasher<Hasher>::value,
+        "Hasher requirements not met");
+    static_assert(is_Progress<Progress>::value,
+        "Progress requirements not met");
+    using namespace detail;
+    auto const readSize = 1024 * block_size(dat_path);
+    auto const writeSize = 16 * block_size(key_path);
+
+    // Open data file for reading and appending
+    File df{args...};
+    df.open(file_mode::append, dat_path, ec);
+    if(ec)
+        return;
+    dat_file_header dh;
+    read(df, dh, ec);
+    if(ec)
+        return;
+    verify(dh, ec);
+    if(ec)
+        return;
+    auto const dataFileSize = df.size(ec);
+    if(ec)
+        return;
+
+    // Make sure log file doesn't exist
+    File lf{args...};
+    lf.open(file_mode::read, log_path, ec);
+    if(! ec)
+        ec = error::log_file_exists;
+    if(ec != errc::no_such_file_or_directory)
+        return;
+    ec = {};
+
+    // Set up key file header
+    key_file_header kh;
+    kh.version = currentVersion;
+    kh.uid = dh.uid;
+    kh.appnum = dh.appnum;
+    kh.key_size = dh.key_size;
+    kh.salt = make_salt();
+    kh.pepper = pepper<Hasher>(kh.salt);
+    kh.block_size = blockSize;
+    kh.load_factor = std::min<std::size_t>(
+        static_cast<std::size_t>(65536.0 * loadFactor), 65535);
+    kh.buckets = static_cast<std::size_t>(
+        std::ceil(itemCount /(
+            bucket_capacity(kh.block_size) * loadFactor)));
+    kh.modulus = ceil_pow2(kh.buckets);
+    // Create key file
+    File kf{args...};
+    kf.create(file_mode::write, key_path, ec);
+    if(ec)
+        return;
+    // Write key file header
+    // Note, file size is less than any valid block_size here
+    {
+        std::array<std::uint8_t, key_file_header::size> buf;
+        ostream os{buf.data(), buf.size()};
+        write(os, kh);
+        kf.write(0, buf.data(), buf.size(), ec);
+        if(ec)
+            return;
+        kf.sync(ec);
+        if(ec)
+            return;
+    }
+
+    // Create log file
+    lf.create(file_mode::append, log_path, ec);
+    if(ec)
+        return;
+    // Write log file header
+    {
+        log_file_header lh;
+        lh.version = currentVersion;            // Version
+        lh.uid = kh.uid;                        // UID
+        lh.appnum = kh.appnum;                  // Appnum
+        lh.key_size = kh.key_size;              // Key Size
+        lh.salt = kh.salt;                      // Salt
+        lh.pepper = pepper<Hasher>(kh.salt);    // Pepper
+        lh.block_size = kh.block_size;          // Block Size
+        lh.key_file_size = 0;                   // Key File Size
+        lh.dat_file_size = dataFileSize;        // Data File Size
+        write(lf, lh, ec);
+        if(ec)
+            return;
+        lf.sync(ec);
+        if(ec)
+            return;
+    }
+
+    // Create full key file
+    buffer buf{kh.block_size};
+    {
+        // Write key file header
+        std::memset(buf.get(), 0, kh.block_size);
+        ostream os{buf.get(), kh.block_size};
+        write(os, kh);
+        kf.write(0, buf.get(), buf.size(), ec);
+        if(ec)
+            return;
+        kf.sync(ec);
+        if(ec)
+            return;
+        // Pre-allocate space for the entire key file
+        std::uint8_t zero = 0;
+        kf.write(
+            static_cast<noff_t>(kh.buckets + 1) * kh.block_size - 1,
+                &zero, 1, ec);
+        if(ec)
+            return;
+        kf.sync(ec);
+        if(ec)
+            return;
+    }
+    
+    // Build contiguous sequential sections of the
+    // key file using multiple passes over the data.
+    //
+    auto const chunkSize = std::max<std::size_t>(1,
+        bufferSize / kh.block_size);
+    // Calculate work required
+    auto const passes =
+       (kh.buckets + chunkSize - 1) / chunkSize;
+    auto const nwork = passes * dataFileSize;
+    progress(0, nwork);
+
+    buf.reserve(chunkSize * kh.block_size);
+    bulk_writer<File> dw{df, dataFileSize, writeSize};
+    for(nbuck_t b0 = 0; b0 < kh.buckets; b0 += chunkSize)
+    {
+        auto const b1 = std::min<std::size_t>(b0 + chunkSize, kh.buckets);
+        // Buffered range is [b0, b1)
+        auto const bn = b1 - b0;
+        // Create empty buckets
+        for(std::size_t i = 0; i < bn; ++i)
+            bucket b{kh.block_size,
+                buf.get() + i * kh.block_size, empty};
+        // Insert all keys into buckets
+        // Iterate Data File
+        bulk_reader<File> r{df,
+            dat_file_header::size, dataFileSize, readSize};
+        while(! r.eof())
+        {
+            auto const offset = r.offset();
+            // Data Record or Spill Record
+            nsize_t size;
+            auto is = r.prepare(
+                field<uint48_t>::size, ec); // Size
+            if(ec)
+                return;
+            progress((b0 / chunkSize) * dataFileSize + r.offset(), nwork);
+            read_size48(is, size);
+            if(size > 0)
+            {
+                // Data Record
+                is = r.prepare(
+                    dh.key_size +           // Key
+                    size, ec);              // Data
+                if(ec)
+                    return;
+                std::uint8_t const* const key =
+                    is.data(dh.key_size);
+                auto const h = hash<Hasher>(
+                    key, dh.key_size, kh.salt);
+                auto const n = bucket_index(
+                    h, kh.buckets, kh.modulus);
+                if(n < b0 || n >= b1)
+                    continue;
+                bucket b{kh.block_size, buf.get() +
+                   (n - b0) * kh.block_size};
+                maybe_spill(b, dw, ec);
+                if(ec)
+                    return;
+                b.insert(offset, size, h);
+            }
+            else
+            {
+                // VFALCO Should never get here
+                // Spill Record
+                is = r.prepare(
+                    field<std::uint16_t>::size, ec);
+                if(ec)
+                    return;
+                read<std::uint16_t>(is, size);  // Size
+                r.prepare(size, ec); // skip
+                if(ec)
+                    return;
+            }
+        }
+        kf.write((b0 + 1) * kh.block_size, buf.get(),
+            static_cast<std::size_t>(bn * kh.block_size), ec);
+        if(ec)
+            return;
+    }
+    dw.flush(ec);
+    if(ec)
+        return;
+    lf.close();
+    File::erase(log_path, ec);
+    if(ec)
+        return;
+}
+
+} // nudb
+
+#endif