@nxtedition/rocksdb 1.0.0

package/deps/rocksdb/rocksdb/file/writable_file_writer.h

@@ -0,0 +1,251 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <atomic>
+#include <string>
+
+#include "db/version_edit.h"
+#include "env/file_system_tracer.h"
+#include "port/port.h"
+#include "rocksdb/env.h"
+#include "rocksdb/file_checksum.h"
+#include "rocksdb/file_system.h"
+#include "rocksdb/io_status.h"
+#include "rocksdb/listener.h"
+#include "rocksdb/rate_limiter.h"
+#include "test_util/sync_point.h"
+#include "util/aligned_buffer.h"
+
+namespace ROCKSDB_NAMESPACE {
+class Statistics;
+
+// WritableFileWriter is a wrapper on top of Env::WritableFile. It provides
+// facilities to:
+// - Handle Buffered and Direct writes.
+// - Rate limit writes.
+// - Flush and Sync the data to the underlying filesystem.
+// - Notify any interested listeners on the completion of a write.
+// - Update IO stats.
+class WritableFileWriter {
+ private:
+#ifndef ROCKSDB_LITE
+  void NotifyOnFileWriteFinish(
+      uint64_t offset, size_t length,
+      const FileOperationInfo::StartTimePoint& start_ts,
+      const FileOperationInfo::FinishTimePoint& finish_ts,
+      const IOStatus& io_status) {
+    FileOperationInfo info(FileOperationType::kWrite, file_name_, start_ts,
+                           finish_ts, io_status);
+    info.offset = offset;
+    info.length = length;
+
+    for (auto& listener : listeners_) {
+      listener->OnFileWriteFinish(info);
+    }
+    info.status.PermitUncheckedError();
+  }
+  void NotifyOnFileFlushFinish(
+      const FileOperationInfo::StartTimePoint& start_ts,
+      const FileOperationInfo::FinishTimePoint& finish_ts,
+      const IOStatus& io_status) {
+    FileOperationInfo info(FileOperationType::kFlush, file_name_, start_ts,
+                           finish_ts, io_status);
+
+    for (auto& listener : listeners_) {
+      listener->OnFileFlushFinish(info);
+    }
+    info.status.PermitUncheckedError();
+  }
+  void NotifyOnFileSyncFinish(
+      const FileOperationInfo::StartTimePoint& start_ts,
+      const FileOperationInfo::FinishTimePoint& finish_ts,
+      const IOStatus& io_status,
+      FileOperationType type = FileOperationType::kSync) {
+    FileOperationInfo info(type, file_name_, start_ts, finish_ts, io_status);
+
+    for (auto& listener : listeners_) {
+      listener->OnFileSyncFinish(info);
+    }
+    info.status.PermitUncheckedError();
+  }
+  void NotifyOnFileRangeSyncFinish(
+      uint64_t offset, size_t length,
+      const FileOperationInfo::StartTimePoint& start_ts,
+      const FileOperationInfo::FinishTimePoint& finish_ts,
+      const IOStatus& io_status) {
+    FileOperationInfo info(FileOperationType::kRangeSync, file_name_, start_ts,
+                           finish_ts, io_status);
+    info.offset = offset;
+    info.length = length;
+
+    for (auto& listener : listeners_) {
+      listener->OnFileRangeSyncFinish(info);
+    }
+    info.status.PermitUncheckedError();
+  }
+  void NotifyOnFileTruncateFinish(
+      const FileOperationInfo::StartTimePoint& start_ts,
+      const FileOperationInfo::FinishTimePoint& finish_ts,
+      const IOStatus& io_status) {
+    FileOperationInfo info(FileOperationType::kTruncate, file_name_, start_ts,
+                           finish_ts, io_status);
+
+    for (auto& listener : listeners_) {
+      listener->OnFileTruncateFinish(info);
+    }
+    info.status.PermitUncheckedError();
+  }
+  void NotifyOnFileCloseFinish(
+      const FileOperationInfo::StartTimePoint& start_ts,
+      const FileOperationInfo::FinishTimePoint& finish_ts,
+      const IOStatus& io_status) {
+    FileOperationInfo info(FileOperationType::kClose, file_name_, start_ts,
+                           finish_ts, io_status);
+
+    for (auto& listener : listeners_) {
+      listener->OnFileCloseFinish(info);
+    }
+    info.status.PermitUncheckedError();
+  }
+#endif  // ROCKSDB_LITE
+
+  bool ShouldNotifyListeners() const { return !listeners_.empty(); }
+  void UpdateFileChecksum(const Slice& data);
+
+  std::string file_name_;
+  FSWritableFilePtr writable_file_;
+  Env* env_;
+  AlignedBuffer buf_;
+  size_t max_buffer_size_;
+  // Actually written data size can be used for truncate
+  // not counting padding data
+  uint64_t filesize_;
+#ifndef ROCKSDB_LITE
+  // This is necessary when we use unbuffered access
+  // and writes must happen on aligned offsets
+  // so we need to go back and write that page again
+  uint64_t next_write_offset_;
+#endif  // ROCKSDB_LITE
+  bool pending_sync_;
+  uint64_t last_sync_size_;
+  uint64_t bytes_per_sync_;
+  RateLimiter* rate_limiter_;
+  Statistics* stats_;
+  std::vector<std::shared_ptr<EventListener>> listeners_;
+  std::unique_ptr<FileChecksumGenerator> checksum_generator_;
+  bool checksum_finalized_;
+
+ public:
+  WritableFileWriter(
+      std::unique_ptr<FSWritableFile>&& file, const std::string& _file_name,
+      const FileOptions& options, Env* env = nullptr,
+      const std::shared_ptr<IOTracer>& io_tracer = nullptr,
+      Statistics* stats = nullptr,
+      const std::vector<std::shared_ptr<EventListener>>& listeners = {},
+      FileChecksumGenFactory* file_checksum_gen_factory = nullptr)
+      : file_name_(_file_name),
+        writable_file_(std::move(file), io_tracer),
+        env_(env),
+        buf_(),
+        max_buffer_size_(options.writable_file_max_buffer_size),
+        filesize_(0),
+#ifndef ROCKSDB_LITE
+        next_write_offset_(0),
+#endif  // ROCKSDB_LITE
+        pending_sync_(false),
+        last_sync_size_(0),
+        bytes_per_sync_(options.bytes_per_sync),
+        rate_limiter_(options.rate_limiter),
+        stats_(stats),
+        listeners_(),
+        checksum_generator_(nullptr),
+        checksum_finalized_(false) {
+    TEST_SYNC_POINT_CALLBACK("WritableFileWriter::WritableFileWriter:0",
+                             reinterpret_cast<void*>(max_buffer_size_));
+    buf_.Alignment(writable_file_->GetRequiredBufferAlignment());
+    buf_.AllocateNewBuffer(std::min((size_t)65536, max_buffer_size_));
+#ifndef ROCKSDB_LITE
+    std::for_each(listeners.begin(), listeners.end(),
+                  [this](const std::shared_ptr<EventListener>& e) {
+                    if (e->ShouldBeNotifiedOnFileIO()) {
+                      listeners_.emplace_back(e);
+                    }
+                  });
+#else  // !ROCKSDB_LITE
+    (void)listeners;
+#endif
+    if (file_checksum_gen_factory != nullptr) {
+      FileChecksumGenContext checksum_gen_context;
+      checksum_gen_context.file_name = _file_name;
+      checksum_generator_ =
+          file_checksum_gen_factory->CreateFileChecksumGenerator(
+              checksum_gen_context);
+    }
+  }
+
+  WritableFileWriter(const WritableFileWriter&) = delete;
+
+  WritableFileWriter& operator=(const WritableFileWriter&) = delete;
+
+  ~WritableFileWriter() {
+    auto s = Close();
+    s.PermitUncheckedError();
+  }
+
+  std::string file_name() const { return file_name_; }
+
+  IOStatus Append(const Slice& data);
+
+  IOStatus Pad(const size_t pad_bytes);
+
+  IOStatus Flush();
+
+  IOStatus Close();
+
+  IOStatus Sync(bool use_fsync);
+
+  // Sync only the data that was already Flush()ed. Safe to call concurrently
+  // with Append() and Flush(). If !writable_file_->IsSyncThreadSafe(),
+  // returns NotSupported status.
+  IOStatus SyncWithoutFlush(bool use_fsync);
+
+  uint64_t GetFileSize() const { return filesize_; }
+
+  IOStatus InvalidateCache(size_t offset, size_t length) {
+    return writable_file_->InvalidateCache(offset, length);
+  }
+
+  FSWritableFile* writable_file() const { return writable_file_.get(); }
+
+  bool use_direct_io() { return writable_file_->use_direct_io(); }
+
+  bool TEST_BufferIsEmpty() { return buf_.CurrentSize() == 0; }
+
+  void TEST_SetFileChecksumGenerator(
+      FileChecksumGenerator* checksum_generator) {
+    checksum_generator_.reset(checksum_generator);
+  }
+
+  std::string GetFileChecksum();
+
+  const char* GetFileChecksumFuncName() const;
+
+ private:
+  // Used when os buffering is OFF and we are writing
+  // DMA such as in Direct I/O mode
+#ifndef ROCKSDB_LITE
+  IOStatus WriteDirect();
+#endif  // !ROCKSDB_LITE
+  // Normal write
+  IOStatus WriteBuffered(const char* data, size_t size);
+  IOStatus RangeSync(uint64_t offset, uint64_t nbytes);
+  IOStatus SyncInternal(bool use_fsync);
+};
+}  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/hdfs/env_hdfs.h

@@ -0,0 +1,386 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+
+#pragma once
+#include <algorithm>
+#include <stdio.h>
+#include <time.h>
+#include <iostream>
+#include "port/sys_time.h"
+#include "rocksdb/env.h"
+#include "rocksdb/status.h"
+
+#ifdef USE_HDFS
+#include <hdfs.h>
+
+namespace ROCKSDB_NAMESPACE {
+
+// Thrown during execution when there is an issue with the supplied
+// arguments.
+class HdfsUsageException : public std::exception { };
+
+// A simple exception that indicates something went wrong that is not
+// recoverable.  The intention is for the message to be printed (with
+// nothing else) and the process terminate.
+class HdfsFatalException : public std::exception {
+public:
+  explicit HdfsFatalException(const std::string& s) : what_(s) { }
+  virtual ~HdfsFatalException() throw() { }
+  virtual const char* what() const throw() {
+    return what_.c_str();
+  }
+private:
+  const std::string what_;
+};
+
+//
+// The HDFS environment for rocksdb. This class overrides all the
+// file/dir access methods and delegates the thread-mgmt methods to the
+// default posix environment.
+//
+class HdfsEnv : public Env {
+
+ public:
+  explicit HdfsEnv(const std::string& fsname) : fsname_(fsname) {
+    posixEnv = Env::Default();
+    fileSys_ = connectToPath(fsname_);
+  }
+
+  virtual ~HdfsEnv() {
+    fprintf(stderr, "Destroying HdfsEnv::Default()\n");
+    hdfsDisconnect(fileSys_);
+  }
+
+  Status NewSequentialFile(const std::string& fname,
+                           std::unique_ptr<SequentialFile>* result,
+                           const EnvOptions& options) override;
+
+  Status NewRandomAccessFile(const std::string& fname,
+                             std::unique_ptr<RandomAccessFile>* result,
+                             const EnvOptions& options) override;
+
+  Status NewWritableFile(const std::string& fname,
+                         std::unique_ptr<WritableFile>* result,
+                         const EnvOptions& options) override;
+
+  Status NewDirectory(const std::string& name,
+                      std::unique_ptr<Directory>* result) override;
+
+  Status FileExists(const std::string& fname) override;
+
+  Status GetChildren(const std::string& path,
+                     std::vector<std::string>* result) override;
+
+  Status DeleteFile(const std::string& fname) override;
+
+  Status CreateDir(const std::string& name) override;
+
+  Status CreateDirIfMissing(const std::string& name) override;
+
+  Status DeleteDir(const std::string& name) override;
+
+  Status GetFileSize(const std::string& fname, uint64_t* size) override;
+
+  Status GetFileModificationTime(const std::string& fname,
+                                 uint64_t* file_mtime) override;
+
+  Status RenameFile(const std::string& src, const std::string& target) override;
+
+  Status LinkFile(const std::string& /*src*/,
+                  const std::string& /*target*/) override {
+    return Status::NotSupported(); // not supported
+  }
+
+  Status LockFile(const std::string& fname, FileLock** lock) override;
+
+  Status UnlockFile(FileLock* lock) override;
+
+  Status NewLogger(const std::string& fname,
+                   std::shared_ptr<Logger>* result) override;
+
+  Status IsDirectory(const std::string& path, bool* is_dir) override;
+
+  void Schedule(void (*function)(void* arg), void* arg, Priority pri = LOW,
+                void* tag = nullptr,
+                void (*unschedFunction)(void* arg) = 0) override {
+    posixEnv->Schedule(function, arg, pri, tag, unschedFunction);
+  }
+
+  int UnSchedule(void* tag, Priority pri) override {
+    return posixEnv->UnSchedule(tag, pri);
+  }
+
+  void StartThread(void (*function)(void* arg), void* arg) override {
+    posixEnv->StartThread(function, arg);
+  }
+
+  void WaitForJoin() override { posixEnv->WaitForJoin(); }
+
+  unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const override {
+    return posixEnv->GetThreadPoolQueueLen(pri);
+  }
+
+  Status GetTestDirectory(std::string* path) override {
+    return posixEnv->GetTestDirectory(path);
+  }
+
+  uint64_t NowMicros() override { return posixEnv->NowMicros(); }
+
+  void SleepForMicroseconds(int micros) override {
+    posixEnv->SleepForMicroseconds(micros);
+  }
+
+  Status GetHostName(char* name, uint64_t len) override {
+    return posixEnv->GetHostName(name, len);
+  }
+
+  Status GetCurrentTime(int64_t* unix_time) override {
+    return posixEnv->GetCurrentTime(unix_time);
+  }
+
+  Status GetAbsolutePath(const std::string& db_path,
+                         std::string* output_path) override {
+    return posixEnv->GetAbsolutePath(db_path, output_path);
+  }
+
+  void SetBackgroundThreads(int number, Priority pri = LOW) override {
+    posixEnv->SetBackgroundThreads(number, pri);
+  }
+
+  int GetBackgroundThreads(Priority pri = LOW) override {
+    return posixEnv->GetBackgroundThreads(pri);
+  }
+
+  void IncBackgroundThreadsIfNeeded(int number, Priority pri) override {
+    posixEnv->IncBackgroundThreadsIfNeeded(number, pri);
+  }
+
+  std::string TimeToString(uint64_t number) override {
+    return posixEnv->TimeToString(number);
+  }
+
+  static uint64_t gettid() { return Env::Default()->GetThreadID(); }
+
+  uint64_t GetThreadID() const override { return HdfsEnv::gettid(); }
+
+ private:
+  std::string fsname_;  // string of the form "hdfs://hostname:port/"
+  hdfsFS fileSys_;      //  a single FileSystem object for all files
+  Env*  posixEnv;       // This object is derived from Env, but not from
+                        // posixEnv. We have posixnv as an encapsulated
+                        // object here so that we can use posix timers,
+                        // posix threads, etc.
+
+  static const std::string kProto;
+  static const std::string pathsep;
+
+  /**
+   * If the URI is specified of the form hdfs://server:port/path,
+   * then connect to the specified cluster
+   * else connect to default.
+   */
+  hdfsFS connectToPath(const std::string& uri) {
+    if (uri.empty()) {
+      return nullptr;
+    }
+    if (uri.find(kProto) != 0) {
+      // uri doesn't start with hdfs:// -> use default:0, which is special
+      // to libhdfs.
+      return hdfsConnectNewInstance("default", 0);
+    }
+    const std::string hostport = uri.substr(kProto.length());
+
+    std::vector <std::string> parts;
+    split(hostport, ':', parts);
+    if (parts.size() != 2) {
+      throw HdfsFatalException("Bad uri for hdfs " + uri);
+    }
+    // parts[0] = hosts, parts[1] = port/xxx/yyy
+    std::string host(parts[0]);
+    std::string remaining(parts[1]);
+
+    int rem = static_cast<int>(remaining.find(pathsep));
+    std::string portStr = (rem == 0 ? remaining :
+                           remaining.substr(0, rem));
+
+    tPort port = static_cast<tPort>(atoi(portStr.c_str()));
+    if (port == 0) {
+      throw HdfsFatalException("Bad host-port for hdfs " + uri);
+    }
+    hdfsFS fs = hdfsConnectNewInstance(host.c_str(), port);
+    return fs;
+  }
+
+  void split(const std::string &s, char delim,
+             std::vector<std::string> &elems) {
+    elems.clear();
+    size_t prev = 0;
+    size_t pos = s.find(delim);
+    while (pos != std::string::npos) {
+      elems.push_back(s.substr(prev, pos));
+      prev = pos + 1;
+      pos = s.find(delim, prev);
+    }
+    elems.push_back(s.substr(prev, s.size()));
+  }
+};
+
+}  // namespace ROCKSDB_NAMESPACE
+
+#else // USE_HDFS
+
+namespace ROCKSDB_NAMESPACE {
+
+class HdfsEnv : public Env {
+
+ public:
+  explicit HdfsEnv(const std::string& /*fsname*/) {
+    fprintf(stderr, "You have not build rocksdb with HDFS support\n");
+    fprintf(stderr, "Please see hdfs/README for details\n");
+    abort();
+  }
+
+  virtual ~HdfsEnv() {
+  }
+
+  virtual Status NewSequentialFile(const std::string& fname,
+                                   std::unique_ptr<SequentialFile>* result,
+                                   const EnvOptions& options) override;
+
+  virtual Status NewRandomAccessFile(
+      const std::string& /*fname*/,
+      std::unique_ptr<RandomAccessFile>* /*result*/,
+      const EnvOptions& /*options*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status NewWritableFile(const std::string& /*fname*/,
+                                 std::unique_ptr<WritableFile>* /*result*/,
+                                 const EnvOptions& /*options*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status NewDirectory(const std::string& /*name*/,
+                              std::unique_ptr<Directory>* /*result*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status FileExists(const std::string& /*fname*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status GetChildren(const std::string& /*path*/,
+                             std::vector<std::string>* /*result*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status DeleteFile(const std::string& /*fname*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status CreateDir(const std::string& /*name*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status CreateDirIfMissing(const std::string& /*name*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status DeleteDir(const std::string& /*name*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status GetFileSize(const std::string& /*fname*/,
+                             uint64_t* /*size*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status GetFileModificationTime(const std::string& /*fname*/,
+                                         uint64_t* /*time*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status RenameFile(const std::string& /*src*/,
+                            const std::string& /*target*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status LinkFile(const std::string& /*src*/,
+                          const std::string& /*target*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status LockFile(const std::string& /*fname*/,
+                          FileLock** /*lock*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status UnlockFile(FileLock* /*lock*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status NewLogger(const std::string& /*fname*/,
+                           std::shared_ptr<Logger>* /*result*/) override {
+    return Status::NotSupported();
+  }
+
+  Status IsDirectory(const std::string& /*path*/, bool* /*is_dir*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual void Schedule(void (* /*function*/)(void* arg), void* /*arg*/,
+                        Priority /*pri*/ = LOW, void* /*tag*/ = nullptr,
+                        void (* /*unschedFunction*/)(void* arg) = 0) override {}
+
+  virtual int UnSchedule(void* /*tag*/, Priority /*pri*/) override { return 0; }
+
+  virtual void StartThread(void (* /*function*/)(void* arg),
+                           void* /*arg*/) override {}
+
+  virtual void WaitForJoin() override {}
+
+  virtual unsigned int GetThreadPoolQueueLen(
+      Priority /*pri*/ = LOW) const override {
+    return 0;
+  }
+
+  virtual Status GetTestDirectory(std::string* /*path*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual uint64_t NowMicros() override { return 0; }
+
+  virtual void SleepForMicroseconds(int /*micros*/) override {}
+
+  virtual Status GetHostName(char* /*name*/, uint64_t /*len*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status GetCurrentTime(int64_t* /*unix_time*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual Status GetAbsolutePath(const std::string& /*db_path*/,
+                                 std::string* /*outputpath*/) override {
+    return Status::NotSupported();
+  }
+
+  virtual void SetBackgroundThreads(int /*number*/,
+                                    Priority /*pri*/ = LOW) override {}
+  virtual int GetBackgroundThreads(Priority /*pri*/ = LOW) override {
+    return 0;
+  }
+  virtual void IncBackgroundThreadsIfNeeded(int /*number*/,
+                                            Priority /*pri*/) override {}
+  virtual std::string TimeToString(uint64_t /*number*/) override { return ""; }
+
+  virtual uint64_t GetThreadID() const override {
+    return 0;
+  }
+};
+}  // namespace ROCKSDB_NAMESPACE
+
+#endif // USE_HDFS