clickhouse-native 0.0.1

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/base/compressed.cpp

@@ -0,0 +1,258 @@
+#include "compressed.h"
+#include "wire_format.h"
+#include "output.h"
+#include "clickhouse/exceptions.h"
+
+#include <city.h>
+#include <lz4.h>
+#include <exception>
+#include <zstd.h>
+#include <stdexcept>
+#include <system_error>
+
+namespace {
+constexpr size_t HEADER_SIZE = 9;
+
+// see DB::CompressionMethodByte from src/Compression/CompressionInfo.h of ClickHouse project
+enum class CompressionMethodByte : uint8_t {
+    NONE = 0x02,
+    LZ4  = 0x82,
+    ZSTD = 0x90,
+};
+
+// Documentation says that compression is faster when output buffer is larger than LZ4_compressBound/ZSTD_compressBound estimation.
+constexpr size_t EXTRA_COMPRESS_BUFFER_SIZE = 4096;
+constexpr size_t DBMS_MAX_COMPRESSED_SIZE = 0x40000000ULL;   // 1GB
+}
+
+namespace clickhouse {
+
+CompressedInput::CompressedInput(InputStream* input)
+    : input_(input)
+{
+}
+
+CompressedInput::~CompressedInput() {
+    if (!mem_.Exhausted()) {
+#if __cplusplus < 201703L
+        if (!std::uncaught_exception()) {
+#else
+        if (!std::uncaught_exceptions()) {
+#endif
+            throw CompressionError("some data was not read");
+        }
+    }
+}
+
+size_t CompressedInput::DoNext(const void** ptr, size_t len) {
+    if (mem_.Exhausted()) {
+        if (!Decompress()) {
+            return 0;
+        }
+    }
+
+    return mem_.Next(ptr, len);
+}
+
+bool CompressedInput::Decompress() {
+    uint128 hash;
+    uint32_t compressed = 0;
+    uint32_t original = 0;
+    uint8_t method = 0;
+
+    if (!WireFormat::ReadFixed(*input_, &hash)) {
+        return false;
+    }
+    if (!WireFormat::ReadFixed(*input_, &method)) {
+        return false;
+    }
+
+    if (method != static_cast<uint8_t>(CompressionMethodByte::LZ4) && method != static_cast<uint8_t>(CompressionMethodByte::ZSTD)) {
+        throw CompressionError("unsupported compression method " + std::to_string((method)));
+    }
+
+    if (!WireFormat::ReadFixed(*input_, &compressed)) {
+        return false;
+    }
+    if (!WireFormat::ReadFixed(*input_, &original)) {
+        return false;
+    }
+
+    if (compressed > DBMS_MAX_COMPRESSED_SIZE) {
+        throw CompressionError("compressed data too big");
+    }
+
+    Buffer tmp(compressed);
+
+    // Data header
+    {
+        BufferOutput out(&tmp);
+        out.Write(&method, sizeof(method));
+        out.Write(&compressed, sizeof(compressed));
+        out.Write(&original, sizeof(original));
+        out.Flush();
+    }
+
+    if (!WireFormat::ReadBytes(*input_, tmp.data() + HEADER_SIZE, compressed - HEADER_SIZE)) {
+        return false;
+    } else {
+        if (hash != CityHash128((const char*)tmp.data(), compressed)) {
+            throw CompressionError("data was corrupted");
+        }
+    }
+
+    data_ = Buffer(original);
+
+    switch (method) {
+    case static_cast<uint8_t>(CompressionMethodByte::LZ4): {
+        if (LZ4_decompress_safe((const char*)tmp.data() + HEADER_SIZE, (char*)data_.data(), static_cast<int>(compressed - HEADER_SIZE), original) < 0) {
+            throw CompressionError("can't decompress LZ4-encoded data");
+        } else {
+            mem_.Reset(data_.data(), original);
+        }
+        return true;
+    }
+
+    case static_cast<uint8_t>(CompressionMethodByte::ZSTD): {
+        size_t res = ZSTD_decompress((char*)data_.data(), original, (const char*)tmp.data() + HEADER_SIZE, static_cast<int>(compressed - HEADER_SIZE));
+
+        if (ZSTD_isError(res)) {
+            throw CompressionError("can't decompress ZSTD-encoded data, ZSTD error: " + std::string(ZSTD_getErrorName(res)));
+        } else {
+            mem_.Reset(data_.data(), original);
+        }
+        return true;
+    }
+
+    case static_cast<uint8_t>(CompressionMethodByte::NONE): {
+        throw CompressionError("compression method not defined" + std::to_string((method)));
+    }
+    default: {
+        throw CompressionError("Unknown or unsupported compression method " + std::to_string((method)));
+    }
+    }
+
+    return true;
+}
+
+
+CompressedOutput::CompressedOutput(OutputStream * destination, size_t max_compressed_chunk_size, CompressionMethod method)
+    : destination_(destination)
+    , max_compressed_chunk_size_(max_compressed_chunk_size)
+    , method_(method)
+{
+    PreallocateCompressBuffer(max_compressed_chunk_size);
+}
+
+CompressedOutput::~CompressedOutput() { }
+
+size_t CompressedOutput::DoWrite(const void* data, size_t len) {
+    const size_t original_len = len;
+    // what if len > max_compressed_chunk_size_ ?
+    const size_t max_chunk_size = max_compressed_chunk_size_ > 0 ? max_compressed_chunk_size_ : len;
+    if (max_chunk_size > max_compressed_chunk_size_) {
+        PreallocateCompressBuffer(len);
+    }
+
+    while (len > 0) {
+        auto to_compress = std::min(len, max_chunk_size);
+        Compress(data, to_compress);
+
+        len -= to_compress;
+        data = reinterpret_cast<const char*>(data) + to_compress;
+    }
+
+    return original_len - len;
+}
+
+void CompressedOutput::DoFlush() {
+    destination_->Flush();
+}
+
+void CompressedOutput::Compress(const void * data, size_t len) {
+    switch (method_) {  
+    case clickhouse::CompressionMethod::LZ4: {
+        const auto compressed_size = LZ4_compress_default(
+                (const char*)data,
+                (char*)compressed_buffer_.data() + HEADER_SIZE,
+                static_cast<int>(len),
+                static_cast<int>(compressed_buffer_.size() - HEADER_SIZE));
+        if (compressed_size <= 0)
+            throw CompressionError("Failed to compress chunk of " + std::to_string(len) + " bytes, "
+                    "LZ4 error: " + std::to_string(compressed_size));
+
+        {
+            auto header = compressed_buffer_.data();
+            WriteUnaligned(header, CompressionMethodByte::LZ4);
+            // Compressed data size with header
+            WriteUnaligned(header + 1, static_cast<uint32_t>(compressed_size + HEADER_SIZE));
+            // Original data size
+            WriteUnaligned(header + 5, static_cast<uint32_t>(len));
+        }
+
+        WireFormat::WriteFixed(*destination_, CityHash128((const char*)compressed_buffer_.data(), compressed_size + HEADER_SIZE));
+        WireFormat::WriteBytes(*destination_, compressed_buffer_.data(), compressed_size + HEADER_SIZE);
+        break;
+    }
+
+    case clickhouse::CompressionMethod::ZSTD: {
+        const size_t compressed_size = ZSTD_compress(
+                (char*)compressed_buffer_.data() + HEADER_SIZE,
+                static_cast<int>(compressed_buffer_.size() - HEADER_SIZE),
+                (const char*)data,
+                static_cast<int>(len),
+                ZSTD_fast);
+        if (ZSTD_isError(compressed_size))
+            throw CompressionError("Failed to compress chunk of " + std::to_string(len) + " bytes, "
+                    "ZSTD error: " + std::string(ZSTD_getErrorName(compressed_size)));
+
+        {
+            auto header = compressed_buffer_.data();
+            WriteUnaligned(header, CompressionMethodByte::ZSTD);
+            // Compressed data size with header
+            WriteUnaligned(header + 1, static_cast<uint32_t>(compressed_size + HEADER_SIZE));
+            // Original data size
+            WriteUnaligned(header + 5, static_cast<uint32_t>(len));
+        }
+
+        WireFormat::WriteFixed(*destination_, CityHash128((const char*)compressed_buffer_.data(), compressed_size + HEADER_SIZE));
+        WireFormat::WriteBytes(*destination_, compressed_buffer_.data(), compressed_size + HEADER_SIZE);
+        break;
+    }
+
+    case clickhouse::CompressionMethod::None: {
+        throw CompressionError("no compression defined");
+    }
+    }
+
+    destination_->Flush();
+}
+
+void CompressedOutput::PreallocateCompressBuffer(size_t input_size) {
+    switch (method_) {  
+    case clickhouse::CompressionMethod::LZ4: {
+        const auto estimated_compressed_buffer_size = LZ4_compressBound(static_cast<int>(input_size));
+        if (estimated_compressed_buffer_size <= 0)
+            throw CompressionError("Failed to estimate compressed buffer size, LZ4 error: " + std::to_string(estimated_compressed_buffer_size));
+
+        compressed_buffer_.resize(estimated_compressed_buffer_size + HEADER_SIZE + EXTRA_COMPRESS_BUFFER_SIZE);
+        break;
+    }
+
+    case clickhouse::CompressionMethod::ZSTD: {
+        const size_t estimated_compressed_buffer_size = ZSTD_compressBound(static_cast<int>(input_size));
+        if (ZSTD_isError(estimated_compressed_buffer_size))
+            throw CompressionError("Failed to estimate compressed buffer size, ZSTD error: " + std::string(ZSTD_getErrorName(estimated_compressed_buffer_size)));
+
+        compressed_buffer_.resize(estimated_compressed_buffer_size + HEADER_SIZE + EXTRA_COMPRESS_BUFFER_SIZE);
+        break;
+    }
+
+    case clickhouse::CompressionMethod::None: {
+        /// do nothing
+        break;
+    }
+    }
+}
+
+}

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/base/compressed.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include "input.h"
+#include "output.h"
+#include "buffer.h"
+
+#include "clickhouse/client.h"
+
+namespace clickhouse {
+
+class CompressedInput : public ZeroCopyInput {
+public:
+    explicit CompressedInput(InputStream* input);
+    ~CompressedInput() override;
+
+protected:
+    size_t DoNext(const void** ptr, size_t len) override;
+
+    bool Decompress();
+
+private:
+    InputStream* const input_;
+
+    Buffer data_;
+    ArrayInput mem_;
+};
+
+class CompressedOutput : public OutputStream {
+public:
+    explicit CompressedOutput(OutputStream* destination, size_t max_compressed_chunk_size = 0, CompressionMethod method = CompressionMethod::LZ4);
+    ~CompressedOutput() override;
+
+protected:
+    size_t DoWrite(const void* data, size_t len) override;
+    void DoFlush() override;
+
+private:
+    void Compress(const void * data, size_t len);
+    void PreallocateCompressBuffer(size_t input_size);
+
+private:
+    OutputStream * destination_;
+    const size_t max_compressed_chunk_size_;
+    Buffer compressed_buffer_;
+    CompressionMethod method_;
+};
+
+}

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/base/endpoints_iterator.cpp

@@ -0,0 +1,20 @@
+#include "endpoints_iterator.h"
+#include <clickhouse/client.h>
+
+namespace clickhouse {
+
+RoundRobinEndpointsIterator::RoundRobinEndpointsIterator(const std::vector<Endpoint>& _endpoints)
+   :  endpoints (_endpoints)
+   , current_index (endpoints.size() - 1ull)
+{
+}
+
+Endpoint RoundRobinEndpointsIterator::Next()
+{
+   current_index = (current_index + 1ull) % endpoints.size();
+   return endpoints[current_index];
+}
+
+RoundRobinEndpointsIterator::~RoundRobinEndpointsIterator() = default;
+
+}

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/base/endpoints_iterator.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "clickhouse/client.h"
+#include <vector>
+
+namespace clickhouse {
+
+struct ClientOptions;
+
+/**
+ * Base class for iterating through endpoints.
+*/
+class EndpointsIteratorBase
+{
+ public:
+   virtual ~EndpointsIteratorBase() = default;
+
+   virtual Endpoint Next() = 0;
+};
+
+class RoundRobinEndpointsIterator : public EndpointsIteratorBase
+{
+ public:
+    explicit RoundRobinEndpointsIterator(const std::vector<Endpoint>& opts);
+    Endpoint Next() override;
+
+    ~RoundRobinEndpointsIterator() override;
+
+ private:
+    const std::vector<Endpoint>& endpoints;
+    size_t current_index;
+};
+
+}

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/base/input.cpp

@@ -0,0 +1,96 @@
+#include "input.h"
+
+#include <algorithm>
+#include <memory.h>
+
+namespace clickhouse {
+
+bool ZeroCopyInput::Skip(size_t bytes) {
+    while (bytes > 0) {
+        const void* ptr;
+        size_t len = Next(&ptr, bytes);
+
+        if (len == 0) {
+            return false;
+        }
+
+        bytes -= len;
+    }
+
+    return true;
+}
+
+size_t ZeroCopyInput::DoRead(void* buf, size_t len) {
+    const void* ptr;
+    size_t result = DoNext(&ptr, len);
+
+    if (result) {
+        memcpy(buf, ptr, result);
+    }
+
+    return result;
+}
+
+ArrayInput::ArrayInput() noexcept
+    : data_(nullptr)
+    , len_(0)
+{
+}
+
+ArrayInput::ArrayInput(const void* buf, size_t len) noexcept
+    : data_(static_cast<const uint8_t*>(buf))
+    , len_(len)
+{
+}
+
+ArrayInput::~ArrayInput() = default;
+
+size_t ArrayInput::DoNext(const void** ptr, size_t len) {
+    len = std::min(len_, len);
+
+    *ptr   = data_;
+    len_  -= len;
+    data_ += len;
+
+    return len;
+}
+
+
+BufferedInput::BufferedInput(std::unique_ptr<InputStream> source, size_t buflen)
+    : source_(std::move(source))
+    , array_input_(nullptr, 0)
+    , buffer_(buflen)
+{
+}
+
+BufferedInput::~BufferedInput() = default;
+
+void BufferedInput::Reset() {
+    array_input_.Reset(nullptr, 0);
+}
+
+size_t BufferedInput::DoNext(const void** ptr, size_t len)  {
+    if (array_input_.Exhausted()) {
+        array_input_.Reset(
+            buffer_.data(), source_->Read(buffer_.data(), buffer_.size())
+        );
+    }
+
+    return array_input_.Next(ptr, len);
+}
+
+size_t BufferedInput::DoRead(void* buf, size_t len) {
+    if (array_input_.Exhausted()) {
+        if (len > buffer_.size() / 2) {
+            return source_->Read(buf, len);
+        }
+
+        array_input_.Reset(
+            buffer_.data(), source_->Read(buffer_.data(), buffer_.size())
+        );
+    }
+
+    return array_input_.Read(buf, len);
+}
+
+}

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/base/input.h

@@ -0,0 +1,103 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+#include <memory>
+
+namespace clickhouse {
+
+class InputStream {
+public:
+    virtual ~InputStream() noexcept (false)
+    { }
+
+    /// Reads one byte from the stream.
+    inline bool ReadByte(uint8_t* byte) {
+        return DoRead(byte, sizeof(uint8_t)) == sizeof(uint8_t);
+    }
+
+    /// Reads some data from the stream.
+    inline size_t Read(void* buf, size_t len) {
+        return DoRead(buf, len);
+    }
+
+    // Skips a number of bytes.  Returns false if an underlying read error occurs.
+    virtual bool Skip(size_t bytes) = 0;
+
+protected:
+    virtual size_t DoRead(void* buf, size_t len) = 0;
+};
+
+
+class ZeroCopyInput : public InputStream {
+public:
+    inline size_t Next(const void** buf, size_t len) {
+        return DoNext(buf, len);
+    }
+
+    bool Skip(size_t bytes) override;
+
+protected:
+    virtual size_t DoNext(const void** ptr, size_t len) = 0;
+
+    size_t DoRead(void* buf, size_t len) override;
+};
+
+
+/**
+ * A ZeroCopyInput stream backed by an in-memory array of bytes.
+ */
+class ArrayInput : public ZeroCopyInput {
+public:
+     ArrayInput() noexcept;
+     ArrayInput(const void* buf, size_t len) noexcept;
+    ~ArrayInput() override;
+
+    /// Number of bytes available in the stream.
+    inline size_t Avail() const noexcept {
+        return len_;
+    }
+
+    /// Current read position in the memory block used by this stream.
+    inline const uint8_t* Data() const noexcept {
+        return data_;
+    }
+
+    /// Whether there is more data in the stream.
+    inline bool Exhausted() const noexcept {
+        return !Avail();
+    }
+
+    inline void Reset(const void* buf, size_t len) noexcept {
+        data_ = static_cast<const uint8_t*>(buf);
+        len_ = len;
+    }
+
+private:
+    size_t DoNext(const void** ptr, size_t len) override;
+
+private:
+    const uint8_t* data_;
+    size_t len_;
+};
+
+
+class BufferedInput : public ZeroCopyInput {
+public:
+    BufferedInput(std::unique_ptr<InputStream> source, size_t buflen = 8192);
+    ~BufferedInput() override;
+
+    void Reset();
+
+protected:
+    size_t DoRead(void* buf, size_t len) override;
+    size_t DoNext(const void** ptr, size_t len) override;
+
+private:
+    std::unique_ptr<InputStream> const source_;
+    ArrayInput array_input_;
+    std::vector<uint8_t> buffer_;
+};
+
+}

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/base/open_telemetry.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "uuid.h"
+
+#include <string>
+
+namespace clickhouse::open_telemetry {
+
+/// See https://www.w3.org/TR/trace-context/ for trace_flags definition
+enum TraceFlags : uint8_t {
+    TRACE_FLAG_NONE    = 0,
+    TRACE_FLAG_SAMPLED = 1,
+};
+
+/// The runtime info we need to create new OpenTelemetry spans.
+struct TracingContext {
+    UUID trace_id{};
+    uint64_t span_id = 0;
+    std::string tracestate;
+    uint8_t trace_flags = TRACE_FLAG_NONE;
+};
+
+}  // namespace clickhouse::open_telemetry

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/base/output.cpp

@@ -0,0 +1,119 @@
+#include "output.h"
+
+#include <algorithm>
+#include <assert.h>
+#include <memory.h>
+
+namespace clickhouse {
+
+size_t ZeroCopyOutput::DoWrite(const void* data, size_t len) {
+    const size_t original_len = len;
+    while (len > 0) {
+        void* ptr;
+        size_t result = DoNext(&ptr, len);
+
+        if (result) {
+            memcpy(ptr, data, result);
+            len -= result;
+            data = static_cast<const uint8_t*>(data) + result;
+        } else {
+            break;
+        }
+    }
+
+    return original_len - len;
+}
+
+
+ArrayOutput::ArrayOutput(void* buf, size_t len)
+    : buf_(static_cast<uint8_t*>(buf))
+    , end_(buf_ + len)
+    , buffer_size_(len)
+{
+}
+
+ArrayOutput::~ArrayOutput() = default;
+
+size_t ArrayOutput::DoNext(void** data, size_t len) {
+    len = std::min(len, Avail());
+
+    *data = buf_;
+    buf_ += len;
+
+    return len;
+}
+
+
+BufferOutput::BufferOutput(Buffer* buf)
+    : buf_(buf)
+    , pos_(0)
+{
+    assert(buf_);
+}
+
+BufferOutput::~BufferOutput()
+{ }
+
+size_t BufferOutput::DoNext(void** data, size_t len) {
+    if (pos_ + len > buf_->size()) {
+        buf_->resize(pos_ + len);
+    }
+
+    *data = buf_->data() + pos_;
+    pos_ += len;
+
+    return len;
+}
+
+
+BufferedOutput::BufferedOutput(std::unique_ptr<OutputStream> destination, size_t buflen)
+    : destination_(std::move(destination))
+    , buffer_(buflen)
+    , array_output_(buffer_.data(), buflen)
+{
+}
+
+BufferedOutput::~BufferedOutput() { }
+
+void BufferedOutput::Reset() {
+    array_output_.Reset(buffer_.data(), buffer_.size());
+}
+
+void BufferedOutput::DoFlush() {
+    if (array_output_.Data() != buffer_.data()) {
+        size_t len = array_output_.Data() - buffer_.data();
+        const uint8_t* buf = buffer_.data();
+        while (len > 0) {
+            const size_t written = destination_->Write(buf, len);
+            buf += written;
+            len -= written;
+        }
+
+        destination_->Flush();
+
+        array_output_.Reset(buffer_.data(), buffer_.size());
+    }
+}
+
+size_t BufferedOutput::DoNext(void** data, size_t len) {
+    if (array_output_.Avail() < len) {
+        Flush();
+    }
+
+    return array_output_.Next(data, len);
+
+}
+
+size_t BufferedOutput::DoWrite(const void* data, size_t len) {
+    if (array_output_.Avail() < len) {
+        Flush();
+
+        if (len > buffer_.size() / 2) {
+            return destination_->Write(data, len);
+        }
+    }
+
+    return array_output_.Write(data, len);
+}
+
+}