clickhouse-native 0.0.1

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/columns/lowcardinality.h

@@ -0,0 +1,221 @@
+#pragma once
+
+#include "column.h"
+#include "numeric.h"
+#include "nullable.h"
+
+#include <functional>
+#include <string>
+#include <unordered_map>
+#include <utility>
+
+namespace clickhouse {
+
+template <typename NestedColumnType>
+class ColumnLowCardinalityT;
+
+namespace details {
+
+/** LowCardinalityHashKey used as key in unique items hashmap to abstract away key value
+ * (type of which depends on dictionary column) and to reduce likelehood of collisions.
+ *
+ * In order to dramatically reduce collision rate, we use 2 different hashes from 2 different hash functions.
+ * First hash is used in hashtable (to calculate item position).
+ * Second one is used as part of key value and accessed via `operator==()` upon collision resolution/detection.
+ */
+using LowCardinalityHashKey = std::pair<std::uint64_t, std::uint64_t>;
+
+struct LowCardinalityHashKeyHash {
+    inline std::size_t operator()(const LowCardinalityHashKey &hash_key) const noexcept {
+        return hash_key.first;
+    }
+};
+
+}
+
+/*
+ * LC column contains an "invisible" default item at the beginning of the collection. [default, ...]
+ * If the nested type is Nullable, it contains a null-item at the beginning and a default item at the second position. [null, default, ...]
+ * Null map is not serialized in LC columns. Instead, nulls are tracked by having an index of 0.
+ * */
+class ColumnLowCardinality : public Column {
+public:
+    using UniqueItems = std::unordered_map<details::LowCardinalityHashKey, size_t /*dictionary index*/, details::LowCardinalityHashKeyHash>;
+
+    template <typename T>
+    friend class ColumnLowCardinalityT;
+
+private:
+    // IMPLEMENTATION NOTE: ColumnLowCardinalityT takes reference to underlying dictionary column object,
+    // so make sure to NOT change address of the dictionary object (with reset(), swap()) or with anything else.
+    ColumnRef dictionary_column_;
+    ColumnRef index_column_;
+    UniqueItems unique_items_map_;
+
+public:
+    ColumnLowCardinality(ColumnLowCardinality&& col) = default;
+    // c-tor makes a deep copy of the dictionary_column.
+    explicit ColumnLowCardinality(ColumnRef dictionary_column);
+    explicit ColumnLowCardinality(std::shared_ptr<ColumnNullable> dictionary_column);
+
+    template <typename T>
+    explicit ColumnLowCardinality(std::shared_ptr<ColumnNullableT<T>> dictionary_column)
+        : ColumnLowCardinality(dictionary_column->template As<ColumnNullable>())
+    {}
+
+    ~ColumnLowCardinality();
+
+    /// Increase the capacity of the column for large block insertion.
+    void Reserve(size_t new_cap) override;
+
+    /// Appends another LowCardinality column to the end of this one, updating dictionary.
+    void Append(ColumnRef /*column*/) override;
+
+    bool LoadPrefix(InputStream* input, size_t rows) override;
+
+    /// Loads column data from input stream.
+    bool LoadBody(InputStream* input, size_t rows) override;
+
+    /// Saves column prefix to output stream.
+    void SavePrefix(OutputStream* output) override;
+
+    /// Saves column data to output stream.
+    void SaveBody(OutputStream* output) override;
+
+    /// Clear column data.
+    void Clear() override;
+
+    /// Returns count of rows in the column.
+    size_t Size() const override;
+
+    /// Makes slice of current column, with compacted dictionary
+    ColumnRef Slice(size_t begin, size_t len) const override;
+    ColumnRef CloneEmpty() const override;
+    void Swap(Column& other) override;
+    ItemView GetItem(size_t index) const override;
+
+    size_t GetDictionarySize() const;
+    TypeRef GetNestedType() const;
+
+protected:
+    std::uint64_t getDictionaryIndex(std::uint64_t item_index) const;
+    void appendIndex(std::uint64_t item_index);
+    void removeLastIndex();
+    ColumnRef GetDictionary();
+
+    void AppendUnsafe(const ItemView &);
+
+private:
+    void Setup(ColumnRef dictionary_column);
+    void AppendNullItem();
+    void AppendDefaultItem();
+
+    Type::Code index_type_code_;
+
+public:
+    static details::LowCardinalityHashKey computeHashKey(const ItemView &);
+};
+
+/** Type-aware wrapper that provides simple convenience interface for accessing/appending individual items.
+ */
+template <typename DictionaryColumnType>
+class ColumnLowCardinalityT : public ColumnLowCardinality {
+
+    DictionaryColumnType& typed_dictionary_;
+    const Type::Code type_;
+
+public:
+    using WrappedColumnType = DictionaryColumnType;
+    // Type this column takes as argument of Append and returns with At() and operator[]
+    using ValueType = typename DictionaryColumnType::ValueType;
+
+    explicit ColumnLowCardinalityT(ColumnLowCardinality&& col)
+        : ColumnLowCardinality(std::move(col))
+        ,  typed_dictionary_(dynamic_cast<DictionaryColumnType &>(*GetDictionary()))
+        ,  type_(GetTypeCode(typed_dictionary_))
+    {
+    }
+
+    template <typename ...Args>
+    explicit ColumnLowCardinalityT(Args &&... args)
+        : ColumnLowCardinalityT(std::make_shared<DictionaryColumnType>(std::forward<Args>(args)...))
+    {}
+
+    // Create LC<T> column from existing T-column, making a deep copy of all contents.
+    explicit ColumnLowCardinalityT(std::shared_ptr<DictionaryColumnType> dictionary_col)
+        : ColumnLowCardinality(dictionary_col)
+        , typed_dictionary_(dynamic_cast<DictionaryColumnType &>(*GetDictionary()))
+        , type_(GetTypeCode(typed_dictionary_))
+    {}
+
+    /// Extended interface to simplify reading/adding individual items.
+
+    /// Returns element at given row number.
+    inline ValueType At(size_t n) const {
+        return typed_dictionary_.At(getDictionaryIndex(n));
+    }
+
+    /// Returns element at given row number.
+    inline ValueType operator [] (size_t n) const {
+        return typed_dictionary_[getDictionaryIndex(n)];
+    }
+
+    // so the non-virtual Append below doesn't shadow Append() from base class when compiled with older compilers.
+    using ColumnLowCardinality::Append;
+
+    inline void Append(const ValueType & value) {
+        if constexpr (IsNullable<WrappedColumnType>) {
+            if (value.has_value()) {
+                AppendUnsafe(ItemView{type_, *value});
+            } else {
+                AppendUnsafe(ItemView{});
+            }
+        } else {
+            AppendUnsafe(ItemView{type_, value});
+        }
+    }
+
+    template <typename T>
+    inline void AppendMany(const T& container) {
+        for (const auto & item : container) {
+            Append(item);
+        }
+    }
+
+    /** Create a ColumnLowCardinalityT from a ColumnLowCardinality, without copying data and offsets, but by
+     * 'stealing' those from `col`.
+     *
+     *  Ownership of column internals is transferred to returned object, original (argument) object
+     *  MUST NOT BE USED IN ANY WAY, it is only safe to dispose it.
+     *
+     *  Throws an exception if `col` is of wrong type, it is safe to use original col in this case.
+     *  This is a static method to make such conversion verbose.
+     */
+    static auto Wrap(ColumnLowCardinality&& col) {
+        return std::make_shared<ColumnLowCardinalityT<WrappedColumnType>>(std::move(col));
+    }
+
+    static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast<ColumnLowCardinality&&>(col))); }
+
+    // Helper to simplify integration with other APIs
+    static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict<ColumnLowCardinality>())); }
+
+    ColumnRef Slice(size_t begin, size_t size) const override {
+        return Wrap(ColumnLowCardinality::Slice(begin, size));
+    }
+
+    ColumnRef CloneEmpty() const override { return Wrap(ColumnLowCardinality::CloneEmpty()); }
+
+private:
+
+    template <typename T>
+    static auto GetTypeCode(T& column) {
+        if constexpr (IsNullable<T>) {
+            return GetTypeCode(*column.Nested()->template AsStrict<typename T::NestedColumnType>());
+        } else {
+            return column.Type()->GetCode();
+        }
+    }
+};
+
+}

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/columns/lowcardinalityadaptor.h

@@ -0,0 +1,63 @@
+#pragma once
+
+#include "column.h"
+#include "lowcardinality.h"
+
+#include <cassert>
+
+namespace clickhouse {
+
+class OutputStream;
+class CodedInputStream;
+
+/** Adapts any ColumnType to be serialized\deserialized as LowCardinality,
+ *  and to be castable to ColumnType via ColumnPtr->As<ColumnType>().
+ *
+ * It helps to ease migration of the old codebases, which can't afford to switch
+ * to using ColumnLowCardinalityT or ColumnLowCardinality directly,
+ * but still want to benefit from smaller on-wire LowCardinality bandwidth footprint.
+ *
+ * Not intended to be used by users directly.
+ *
+ * @see ClientOptions, CreateColumnByType
+ */
+template <typename AdaptedColumnType>
+class
+[[deprecated("Makes implementation of LC(X) harder and code uglier. Will be removed in next major release (3.0) ")]]
+LowCardinalitySerializationAdaptor : public AdaptedColumnType
+{
+public:
+    using AdaptedColumnType::AdaptedColumnType;
+
+    bool LoadPrefix(InputStream* input, size_t rows) override {
+        auto new_data_column = this->Slice(0, 0)->template As<AdaptedColumnType>();
+        ColumnLowCardinalityT<AdaptedColumnType> low_cardinality_col(new_data_column);
+
+        return low_cardinality_col.LoadPrefix(input, rows);
+    }
+
+    /// Loads column data from input stream.
+    bool LoadBody(InputStream* input, size_t rows) override {
+        auto new_data_column = this->CloneEmpty()->template As<AdaptedColumnType>();
+
+        ColumnLowCardinalityT<AdaptedColumnType> low_cardinality_col(new_data_column);
+        if (!low_cardinality_col.LoadBody(input, rows))
+            return false;
+
+        // It safe to reuse `flat_data_column` later since ColumnLowCardinalityT makes a deep copy, but still check just in case.
+        assert(new_data_column->Size() == 0);
+
+        for (size_t i = 0; i < low_cardinality_col.Size(); ++i)
+            new_data_column->Append(low_cardinality_col[i]);
+
+        this->Swap(*new_data_column);
+        return true;
+    }
+
+    /// Saves column data to output stream.
+    void SaveBody(OutputStream* output) override {
+        ColumnLowCardinalityT<AdaptedColumnType>(this->template As<AdaptedColumnType>()).SaveBody(output);
+    }
+};
+
+}

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/columns/map.cpp

@@ -0,0 +1,87 @@
+#include "map.h"
+
+#include <stdexcept>
+
+#include "../exceptions.h"
+#include "utils.h"
+
+namespace {
+
+using namespace clickhouse;
+
+TypeRef GetMapType(const Type& data_type) {
+    auto array = data_type.As<ArrayType>();
+    if (!array) {
+        throw ValidationError("Wrong type  " + data_type.GetName() + " of data for map");
+    }
+    auto tuple = array->GetItemType()->As<TupleType>();
+    if (!tuple) {
+        throw ValidationError("Wrong type  " + data_type.GetName() + " of data for map");
+    }
+    auto types = tuple->GetTupleType();
+    if (types.size() != 2) {
+        throw ValidationError("Wrong type  " + data_type.GetName() + " of data for map");
+    }
+    return Type::CreateMap(types[0], types[1]);
+}
+
+}  // namespace
+
+namespace clickhouse {
+
+ColumnMap::ColumnMap(ColumnRef data)
+    : Column(GetMapType(data->GetType())), data_(data->As<ColumnArray>()) {
+}
+
+void ColumnMap::Reserve(size_t new_cap) {
+    data_->Reserve(new_cap);
+}
+
+void ColumnMap::Clear() {
+    data_->Clear();
+}
+
+void ColumnMap::Append(ColumnRef column) {
+    if (auto col = column->As<ColumnMap>()) {
+        data_->Append(col->data_);
+    }
+}
+
+bool ColumnMap::LoadPrefix(InputStream* input, size_t rows) {
+    return data_->LoadPrefix(input, rows);
+}
+
+bool ColumnMap::LoadBody(InputStream* input, size_t rows) {
+    return data_->LoadBody(input, rows);
+}
+
+void ColumnMap::SavePrefix(OutputStream* output) {
+    data_->SavePrefix(output);
+}
+
+void ColumnMap::SaveBody(OutputStream* output) {
+    data_->SaveBody(output);
+}
+
+size_t ColumnMap::Size() const {
+    return data_->Size();
+}
+
+ColumnRef ColumnMap::Slice(size_t begin, size_t len) const {
+    return std::make_shared<ColumnMap>(data_->Slice(begin, len));
+}
+
+ColumnRef ColumnMap::CloneEmpty() const {
+    return std::make_shared<ColumnMap>(data_->CloneEmpty());
+}
+
+void ColumnMap::Swap(Column& other) {
+    auto& col = dynamic_cast<ColumnMap&>(other);
+    data_.swap(col.data_);
+}
+
+ColumnRef ColumnMap::GetAsColumn(size_t n) const {
+    return data_->GetAsColumn(n);
+}
+
+}  // namespace clickhouse

data/ext/clickhouse_native/vendor/clickhouse-cpp/clickhouse/columns/map.h

@@ -0,0 +1,257 @@
+#pragma once
+
+#include "../base/projected_iterator.h"
+#include "array.h"
+#include "column.h"
+#include "tuple.h"
+
+#include <functional>
+#include <map>
+
+namespace clickhouse {
+
+template <typename KeyColumnType, typename ValueColumnType>
+class ColumnMapT;
+
+/**
+ * Represents column of Map(K, V).
+ */
+class ColumnMap : public Column {
+public:
+    /** Create a map of given type, with actual values and offsets.
+     *
+     *  Both `data` and `offsets` are used (and modified) internally bye ColumnArray.
+     *  Users are strongly advised against modifying contents of `data` or `offsets` afterwards.
+     */
+    explicit ColumnMap(ColumnRef data);
+
+    /// Increase the capacity of the column for large block insertion.
+    void Reserve(size_t new_cap) override;
+
+    /// Appends content of given column to the end of current one.
+    void Append(ColumnRef column) override;
+
+    /// Loads column prefix from input stream.
+    bool LoadPrefix(InputStream* input, size_t rows) override;
+
+    /// Loads column data from input stream.
+    bool LoadBody(InputStream* input, size_t rows) override;
+
+    /// Saves column prefix to output stream.
+    void SavePrefix(OutputStream* output) override;
+
+    /// Saves column data to output stream.
+    void SaveBody(OutputStream* output) override;
+
+    /// Clear column data .
+    void Clear() override;
+
+    /// Returns count of rows in the column.
+    size_t Size() const override;
+
+    /// Makes slice of the current column.
+    ColumnRef Slice(size_t, size_t) const override;
+    ColumnRef CloneEmpty() const override;
+    void Swap(Column&) override;
+
+    /// Converts map at pos n to column.
+    /// Type of row is tuple {key, value}.
+    ColumnRef GetAsColumn(size_t n) const;
+
+protected:
+    template <typename K, typename V>
+    friend class ColumnMapT;
+
+    ColumnMap(ColumnMap&& map);
+
+private:
+    std::shared_ptr<ColumnArray> data_;
+};
+
+template <typename K, typename V>
+class ColumnMapT : public ColumnMap {
+public:
+    using KeyColumnType   = K;
+    using ValueColumnType = V;
+    using Key             = std::decay_t<decltype(std::declval<KeyColumnType>().At(0))>;
+    using Value           = std::decay_t<decltype(std::declval<ValueColumnType>().At(0))>;
+    using TupleColumnType = ColumnTupleT<KeyColumnType, ValueColumnType>;
+    using ArrayColumnType = ColumnArrayT<TupleColumnType>;
+
+    ColumnMapT(ColumnRef data)
+        : ColumnMap(data), typed_data_(data->AsStrict<ColumnArrayT<TupleColumnType>>()) {}
+
+    ColumnMapT(std::shared_ptr<KeyColumnType> keys, std::shared_ptr<ValueColumnType> values)
+        : ColumnMap(std::make_shared<ArrayColumnType>(std::make_shared<TupleColumnType>(
+              std::make_tuple(std::move(keys), std::move(values))))),
+          typed_data_(data_->template As<ArrayColumnType>()) {}
+
+    ColumnRef Slice(size_t begin, size_t len) const override {
+        return std::make_shared<ColumnMapT<K, V>>(typed_data_->Slice(begin, len));
+    }
+
+    ColumnRef CloneEmpty() const override {
+        return std::make_shared<ColumnMapT<K, V>>(typed_data_->CloneEmpty());
+    }
+
+    void Swap(Column& other) override {
+        auto& col = dynamic_cast<ColumnMapT<K, V>&>(other);
+        col.typed_data_.swap(typed_data_);
+        ColumnMap::Swap(other);
+    }
+
+    /// A single (row) value of the Map-column i.e. read-only map.
+    /// It has a linear time complexity to access items
+    /// Because data base type has same structure
+    /// "This lookup works now with a linear complexity."
+    /// https://clickhouse.com/docs/en/sql-reference/data-types/map
+    /// Convert it to a suitable container required to access more than one element
+
+    class MapValueView {
+        const typename ArrayColumnType::ArrayValueView data_;
+
+    public:
+        using ValueType = std::pair<Key, Value>;
+
+        MapValueView(typename ArrayColumnType::ArrayValueView data) : data_(std::move(data)) {}
+
+        inline auto operator[](const Key& key) const { return (*Find(key)).second; }
+
+        inline auto At(const Key& key) const {
+            auto it = Find(key);
+            if (it == end()) throw ValidationError("ColumnMap value key not found");
+            return (*it).second;
+        }
+
+        class Iterator {
+            typename ArrayColumnType::ArrayValueView::Iterator data_iterator_;
+
+        public:
+            Iterator() = default;
+
+            Iterator(typename ArrayColumnType::ArrayValueView::Iterator data_iterator)
+                : data_iterator_(data_iterator) {}
+
+            using ValueType = std::pair<Key, Value>;
+            using difference_type = size_t;
+            using value_type = ValueType;
+            using pointer = void;
+            using reference = ValueType&;
+            using iterator_category = std::forward_iterator_tag;
+
+            inline auto operator*() const {
+                auto tuple = *data_iterator_;
+                return ValueType{std::get<0>(tuple), std::get<1>(tuple)};
+            }
+
+            inline Iterator& operator++() {
+                ++data_iterator_;
+                return *this;
+            }
+
+            inline bool operator==(const Iterator& other) const {
+                return this->data_iterator_ == other.data_iterator_;
+            }
+
+            inline bool operator!=(const Iterator& other) const { return !(*this == other); }
+        };
+
+        // minimalistic stl-like container interface, hence the lowercase
+        inline Iterator begin() const { return Iterator{data_.begin()}; }
+
+        inline Iterator cbegin() const { return Iterator{data_.cbegin()}; }
+
+        inline Iterator end() const { return Iterator{data_.end()}; }
+
+        inline Iterator cend() const { return Iterator{data_.cend()}; }
+
+        inline size_t size() const { return data_.size(); }
+
+        // It is ugly to have both size() and Size(), but it is for compatitability with both STL
+        // and rest of the clickhouse-cpp.
+        inline size_t Size() const { return data_.Size(); }
+
+        inline size_t Count(const Key& key) const {
+            size_t result = 0;
+            for (auto item : data_) {
+                if (std::get<0>(item) == key) {
+                    ++result;
+                }
+            }
+            return result;
+        }
+
+        inline Iterator Find(const Key& key) const {
+            for (auto it = data_.begin(); it != data_.end(); ++it) {
+                if (std::get<0>(*it) == key) {
+                    return Iterator{it};
+                }
+            }
+            return end();
+        }
+
+        inline bool operator==(const MapValueView& other) const {
+            if (size() != other.size()) {
+                return false;
+            }
+            const auto make_index = [](const auto& data) {
+                std::vector<size_t> result{data.Size()};
+                std::generate(result.begin(), result.end(), [i = 0] () mutable {return i++;});
+                std::sort(result.begin(), result.end(), [&data](size_t l, size_t r) {return data[l] < data[r];});
+                return result;
+            };
+            const auto index = make_index(data_);
+            for (const auto& val : other.data_) {
+                if (!std::binary_search(index.begin(), index.end(), val,
+                        [&data = data_](const auto& l, size_t r) {return l < data[r];})) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        inline bool operator!=(const MapValueView& other) const { return !(*this == other); }
+    };
+
+    inline auto At(size_t index) const { return MapValueView{typed_data_->At(index)}; }
+
+    inline auto operator[](size_t index) const { return At(index); }
+
+    using ColumnMap::Append;
+
+    inline void Append(const MapValueView& value) { typed_data_->Append(value.data_); }
+
+    inline void Append(const std::vector<std::tuple<Key, Value>>& tuples) {
+        typed_data_->Append(tuples.begin(), tuples.end());
+    }
+
+    template <typename T>
+    inline void Append(const T& value) {
+        using BaseIter = decltype(value.begin());
+        using KeyOfT = decltype(std::declval<BaseIter>()->first);
+        using ValOfT = decltype(std::declval<BaseIter>()->second);
+        using Functor = std::function<std::tuple<KeyOfT, ValOfT>(const BaseIter&)>;
+        using Iterator = ProjectedIterator<Functor, BaseIter>;
+
+        Functor functor = [](const BaseIter& i) {
+            return std::make_tuple(std::cref(i->first), std::cref(i->second));
+        };
+
+        typed_data_->Append(Iterator{value.begin(), functor}, Iterator{value.end(), functor});
+    }
+
+    static auto Wrap(ColumnMap&& col) {
+        auto data = ArrayColumnType::Wrap(std::move(col.data_));
+        return std::make_shared<ColumnMapT<K, V>>(std::move(data));
+    }
+
+    static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast<ColumnMap&&>(col))); }
+
+    // Helper to simplify integration with other APIs
+    static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict<ColumnMap>())); }
+
+private:
+    std::shared_ptr<ArrayColumnType> typed_data_;
+};
+
+}  // namespace clickhouse