catboost 1.25.1 → 1.26.0

package/inc/util/stream/tee.h

@@ -1,28 +0,0 @@
-#pragma once
-
-#include "output.h"
-
-/**
- * @addtogroup Streams_Multi
- * @{
- */
-
-/**
- * A proxy output stream that writes into two slave streams simultaneously.
- */
-class TTeeOutput: public IOutputStream {
-public:
-    TTeeOutput(IOutputStream* l, IOutputStream* r) noexcept;
-    ~TTeeOutput() override;
-
-private:
-    void DoWrite(const void* buf, size_t len) override;
-    void DoFlush() override;
-    void DoFinish() override;
-
-private:
-    IOutputStream* L_;
-    IOutputStream* R_;
-};
-
-/** @} */

package/inc/util/stream/tempbuf.h

@@ -1,21 +0,0 @@
-#pragma once
-
-#include "output.h"
-
-#include <util/memory/tempbuf.h>
-
-class TTempBufOutput: public IOutputStream, public TTempBuf {
-public:
-    inline TTempBufOutput() = default;
-
-    explicit TTempBufOutput(size_t size)
-        : TTempBuf(size)
-    {
-    }
-
-    TTempBufOutput(TTempBufOutput&&) noexcept = default;
-    TTempBufOutput& operator=(TTempBufOutput&&) noexcept = default;
-
-protected:
-    void DoWrite(const void* data, size_t len) override;
-};

package/inc/util/stream/tokenizer.h

@@ -1,214 +0,0 @@
-#pragma once
-
-#include "input.h"
-
-#include <util/generic/buffer.h>
-#include <util/generic/mem_copy.h>
-#include <util/generic/strbuf.h>
-#include <util/system/compiler.h>
-#include <util/system/yassert.h>
-
-/**
- * @addtogroup Streams
- * @{
- */
-
-/**
- * Simple stream tokenizer. Splits the stream into tokens that are available
- * via iterator interface.
- *
- * @tparam TEndOfToken                  Predicate for token delimiter characters.
- * @see TEol
- */
-template <typename TEndOfToken>
-class TStreamTokenizer {
-public:
-    class TIterator {
-    public:
-        inline TIterator(TStreamTokenizer* const parent)
-            : Parent_(parent)
-            , AtEnd_(!Parent_->Next(Data_, Len_))
-        {
-        }
-
-        inline TIterator() noexcept
-            : Parent_(nullptr)
-            , Data_(nullptr)
-            , Len_(0)
-            , AtEnd_(true)
-        {
-        }
-
-        inline ~TIterator() = default;
-
-        inline void operator++() {
-            Next();
-        }
-
-        inline bool operator==(const TIterator& l) const noexcept {
-            return AtEnd_ == l.AtEnd_;
-        }
-
-        inline bool operator!=(const TIterator& l) const noexcept {
-            return !(*this == l);
-        }
-
-        /**
-         * @return          Return null-terminated character array with current token.
-         *                  The pointer may be invalid after iterator increment.
-         */
-        inline const char* Data() const noexcept {
-            Y_ASSERT(!AtEnd_);
-
-            return Data_;
-        }
-
-        /**
-         * @return          Length of current token.
-         */
-        inline size_t Length() const noexcept {
-            Y_ASSERT(!AtEnd_);
-
-            return Len_;
-        }
-
-        inline TIterator* operator->() noexcept {
-            return this;
-        }
-
-        inline TStringBuf operator*() noexcept {
-            return TStringBuf{Data_, Len_};
-        }
-
-    private:
-        inline void Next() {
-            Y_ASSERT(Parent_);
-
-            AtEnd_ = !Parent_->Next(Data_, Len_);
-        }
-
-    private:
-        TStreamTokenizer* const Parent_;
-        char* Data_;
-        size_t Len_;
-        bool AtEnd_;
-    };
-
-    inline TStreamTokenizer(IInputStream* const input, const TEndOfToken& eot = TEndOfToken(),
-                            const size_t initial = 1024)
-        : Input_(input)
-        , Buf_(initial)
-        , Cur_(BufBegin())
-        , End_(BufBegin())
-        , Eot_(eot)
-    {
-        CheckBuf();
-    }
-
-    inline bool Next(char*& buf, size_t& len) {
-        char* it = Cur_;
-
-        while (true) {
-            do {
-                while (it != End_) {
-                    if (Eot_(*it)) {
-                        *it = '\0';
-
-                        buf = Cur_;
-                        len = it - Cur_;
-                        Cur_ = it + 1;
-
-                        return true;
-                    } else {
-                        ++it;
-                    }
-                }
-
-                if (Fill() == 0 && End_ != BufEnd()) {
-                    *it = '\0';
-
-                    buf = Cur_;
-                    len = it - Cur_;
-                    Cur_ = End_;
-
-                    return len;
-                }
-            } while (it != BufEnd());
-
-            Y_ASSERT(it == BufEnd());
-            Y_ASSERT(End_ == BufEnd());
-
-            const size_t blen = End_ - Cur_;
-            if (Cur_ == BufBegin()) {
-                Y_ASSERT(blen == Buf_.Capacity());
-
-                /*
-                 * do reallocate
-                 */
-
-                Buf_.Reserve(Buf_.Capacity() * 4);
-                CheckBuf();
-            } else {
-                /*
-                 * do move
-                 */
-
-                MemMove(BufBegin(), Cur_, blen);
-            }
-
-            Cur_ = BufBegin();
-            End_ = Cur_ + blen;
-            it = End_;
-        }
-    }
-
-    inline TIterator begin() {
-        return TIterator{this};
-    }
-
-    inline TIterator end() noexcept {
-        return {};
-    }
-
-private:
-    inline size_t Fill() {
-        const size_t avail = BufEnd() - End_;
-        const size_t readed = Input_->Read(End_, avail);
-
-        End_ += readed;
-
-        return readed;
-    }
-
-    inline char* BufBegin() noexcept {
-        return Buf_.Data();
-    }
-
-    inline char* BufEnd() noexcept {
-        return Buf_.Data() + Buf_.Capacity();
-    }
-
-    inline void CheckBuf() const {
-        if (!Buf_.Data()) {
-            throw std::bad_alloc();
-        }
-    }
-
-private:
-    IInputStream* const Input_;
-    TBuffer Buf_;
-    char* Cur_;
-    char* End_;
-    TEndOfToken Eot_;
-};
-
-/**
- * Predicate for `TStreamTokenizer` that uses '\\n' as a delimiter.
- */
-struct TEol {
-    inline bool operator()(char ch) const noexcept {
-        return ch == '\n';
-    }
-};
-
-/** @} */

package/inc/util/stream/trace.h

@@ -1,60 +0,0 @@
-#pragma once
-
-#include "debug.h"
-
-/**
- * Debug level, as set via `DBGOUT` environment variable.
- */
-enum ETraceLevel : ui8 {
-    TRACE_ERR = 1,
-    TRACE_WARN = 2,
-    TRACE_NOTICE = 3,
-    TRACE_INFO = 4,
-    TRACE_DEBUG = 5,
-    TRACE_DETAIL = 6,
-    TRACE_VERBOSE = 7
-};
-
-#if !defined(NDEBUG) && !defined(Y_ENABLE_TRACE)
-#define Y_ENABLE_TRACE
-#endif
-
-#ifdef Y_ENABLE_TRACE
-
-/**
- * Writes the given data into standard debug stream if current debug level set
- * via `DBGOUT` environment variable permits it.
- *
- * Does nothing in release builds unless `Y_ENABLE_TRACE` is defined.
- *
- * Example usage:
- * @code
- * Y_DBGTRACE(DEBUG, "Advance from " << node1 << " to " << node2);
- * @endcode
- *
- * @param elevel                        Debug level of this trace command, e.g.
- *                                      `WARN` or `DEBUG`. Basically a suffix of
- *                                      one of the values of `ETraceLevel` enum.
- * @param args                          Argument chain to be written out into
- *                                      standard debug stream, joined with `<<`
- *                                      operator.
- * @see ETraceLevel
- */
-#define Y_DBGTRACE(elevel, args) Y_DBGTRACE0(int(TRACE_##elevel), args)
-#define Y_DBGTRACE0(level, args)            \
-    do                                      \
-        if ((level) <= StdDbgLevel()) {     \
-            StdDbgStream() << args << Endl; \
-        }                                   \
-    while (false)
-
-#else
-
-#define Y_DBGTRACE(elevel, args) \
-    do {                         \
-    } while (false)
-#define Y_DBGTRACE0(level, args) \
-    do {                         \
-    } while (false)
-
-#endif

package/inc/util/stream/walk.h

@@ -1,35 +0,0 @@
-#pragma once
-
-#include "zerocopy.h"
-
-/**
- * Zero-copy stream that simplifies implementation of derived classes.
- *
- * Derived classes must implement `DoUnboundedNext` method.
- */
-class IWalkInput: public IZeroCopyInputFastReadTo {
-public:
-    IWalkInput()
-        : Buf_(nullptr)
-        , Len_(0)
-    {
-    }
-
-protected:
-    void DoUndo(size_t len) override;
-    size_t DoNext(const void** ptr, size_t len) override;
-
-    /**
-     * Returns the next data chunk from this input stream. There are no
-     * restrictions on the size of the data chunk.
-     *
-     * @param ptr[out]                  Pointer to the start of the data chunk.
-     * @returns                         Size of the returned data chunk, in bytes.
-     *                                  Return value of zero signals end of stream.
-     */
-    virtual size_t DoUnboundedNext(const void** ptr) = 0;
-
-private:
-    const void* Buf_;
-    size_t Len_;
-};

package/inc/util/stream/zerocopy.h

@@ -1,91 +0,0 @@
-#pragma once
-
-#include <util/system/yassert.h>
-#include <util/system/defaults.h>
-#include <util/generic/ylimits.h>
-
-#include "input.h"
-
-class IOutputStream;
-
-/**
- * @addtogroup Streams
- * @{
- */
-
-/**
- * Input stream with direct access to the input buffer.
- *
- * Derived classes must implement `DoNext` method.
- */
-class IZeroCopyInput: public IInputStream {
-public:
-    IZeroCopyInput() noexcept = default;
-    ~IZeroCopyInput() override;
-
-    IZeroCopyInput(IZeroCopyInput&&) noexcept = default;
-    IZeroCopyInput& operator=(IZeroCopyInput&&) noexcept = default;
-
-    /**
-     * Returns the next data chunk from this input stream.
-     *
-     * Note that this function is not guaranteed to return the requested number
-     * of bytes, even if they are in fact available in the stream.
-     *
-     * @param ptr[out]                  Pointer to the start of the data chunk.
-     * @param len[in]                   Maximal size of the data chunk to be returned, in bytes.
-     * @returns                         Size of the returned data chunk, in bytes.
-     *                                  Return value of zero signals end of stream.
-     */
-    template <class T>
-    inline size_t Next(T** ptr, size_t len) {
-        Y_ASSERT(ptr);
-
-        return DoNext((const void**)ptr, len);
-    }
-
-    template <class T>
-    inline size_t Next(T** ptr) {
-        return Next(ptr, Max<size_t>());
-    }
-
-protected:
-    size_t DoRead(void* buf, size_t len) override;
-    size_t DoSkip(size_t len) override;
-    ui64 DoReadAll(IOutputStream& out) override;
-    virtual size_t DoNext(const void** ptr, size_t len) = 0;
-};
-
-/**
-* Input stream with direct access to the input buffer and ability to undo read
-*
-* Derived classes must implement `DoUndo` method.
-*/
-class IZeroCopyInputFastReadTo: public IZeroCopyInput {
-public:
-    IZeroCopyInputFastReadTo() noexcept = default;
-    ~IZeroCopyInputFastReadTo() override;
-
-    IZeroCopyInputFastReadTo(IZeroCopyInputFastReadTo&&) noexcept = default;
-    IZeroCopyInputFastReadTo& operator=(IZeroCopyInputFastReadTo&&) noexcept = default;
-
-protected:
-    size_t DoReadTo(TString& st, char ch) override;
-
-private:
-    /**
-    * Undo read.
-    *
-    * Note that this function not check if you try undo more that read. In fact Undo used for undo read in last chunk.
-    *
-    * @param len[in]                   Bytes to undo.
-    */
-    inline void Undo(size_t len) {
-        if (len) {
-            DoUndo(len);
-        }
-    }
-    virtual void DoUndo(size_t len) = 0;
-};
-
-/** @} */

package/inc/util/stream/zerocopy_output.h

@@ -1,57 +0,0 @@
-#pragma once
-
-#include <util/system/yassert.h>
-
-#include "output.h"
-
-/**
- * @addtogroup Streams
- * @{
- */
-
-/**
- * Output stream with direct access to the output buffer.
- *
- * Derived classes must implement `DoNext` and `DoUndo` methods.
- */
-class IZeroCopyOutput: public IOutputStream {
-public:
-    IZeroCopyOutput() noexcept = default;
-    ~IZeroCopyOutput() override = default;
-
-    IZeroCopyOutput(IZeroCopyOutput&&) noexcept = default;
-    IZeroCopyOutput& operator=(IZeroCopyOutput&&) noexcept = default;
-
-    /**
-     * Returns the next buffer to write to from this output stream.
-     *
-     * @param ptr[out]                  Pointer to the start of the buffer.
-     * @returns                         Size of the returned buffer, in bytes.
-     *                                  Return value is always nonzero.
-     */
-    template <class T>
-    inline size_t Next(T** ptr) {
-        Y_ASSERT(ptr);
-
-        return DoNext((void**)ptr);
-    }
-
-    /**
-     * Tells the stream that `len` bytes at the end of the buffer returned previously
-     * by Next were actually not written so the current position in stream must be moved backwards.
-     * `len` must not be greater than the size of the buffer previously returned by `Next`.
-     *
-     * @param len[in]                  Number of bytes at the end to move the position by.
-     *
-     */
-    inline void Undo(size_t len) {
-        return DoUndo(len);
-    }
-
-protected:
-    void DoWrite(const void* buf, size_t len) override;
-    virtual size_t DoNext(void** ptr) = 0;
-    virtual void DoUndo(size_t len) = 0;
-};
-
-/** @} */

package/inc/util/stream/zlib.h

@@ -1,173 +0,0 @@
-#pragma once
-
-#include "fwd.h"
-#include "input.h"
-#include "output.h"
-#include "buffered.h"
-
-#include <util/system/defaults.h>
-#include <util/generic/ptr.h>
-#include <util/generic/yexception.h>
-
-/**
- * @addtogroup Streams_Archs
- * @{
- */
-
-struct TZLibError: public yexception {
-};
-
-struct TZLibCompressorError: public TZLibError {
-};
-
-struct TZLibDecompressorError: public TZLibError {
-};
-
-namespace ZLib {
-    enum StreamType : ui8 {
-        Auto = 0, /**< Auto detect format. Can be used for decompression only. */
-        ZLib = 1,
-        GZip = 2,
-        Raw = 3,
-        Invalid = 4
-    };
-
-    enum {
-        ZLIB_BUF_LEN = 8 * 1024
-    };
-}
-
-/**
- * Non-buffered ZLib decompressing stream.
- *
- * Please don't use `TZLibDecompress` if you read text data from stream using
- * `ReadLine`, it is VERY slow (approx 10 times slower, according to synthetic
- * benchmark). For fast buffered ZLib stream reading use `TBufferedZLibDecompress`
- * aka `TZDecompress`.
- */
-class TZLibDecompress: public IInputStream {
-public:
-    TZLibDecompress(IZeroCopyInput* input, ZLib::StreamType type = ZLib::Auto, TStringBuf dict = {});
-    TZLibDecompress(IInputStream* input, ZLib::StreamType type = ZLib::Auto, size_t buflen = ZLib::ZLIB_BUF_LEN,
-                    TStringBuf dict = {});
-
-    /**
-     * Allows/disallows multiple sequential compressed streams. Allowed by default.
-     *
-     * If multiple streams are allowed, their decompressed content will be concatenated.
-     * If multiple streams are disabled, then only first stream is decompressed. After that end
-     * of IInputStream will have happen, i.e. method Read() will return 0.
-     *
-     * @param allowMultipleStreams - flag to allow (true) or disable (false) multiple streams.
-     */
-    void SetAllowMultipleStreams(bool allowMultipleStreams);
-
-    ~TZLibDecompress() override;
-
-protected:
-    size_t DoRead(void* buf, size_t size) override;
-
-public:
-    class TImpl;
-    THolder<TImpl> Impl_;
-};
-
-/**
- * Non-buffered ZLib compressing stream.
- */
-class TZLibCompress: public IOutputStream {
-public:
-    struct TParams {
-        inline TParams(IOutputStream* out)
-            : Out(out)
-            , Type(ZLib::ZLib)
-            , CompressionLevel(6)
-            , BufLen(ZLib::ZLIB_BUF_LEN)
-        {
-        }
-
-        inline TParams& SetType(ZLib::StreamType type) noexcept {
-            Type = type;
-
-            return *this;
-        }
-
-        inline TParams& SetCompressionLevel(size_t level) noexcept {
-            CompressionLevel = level;
-
-            return *this;
-        }
-
-        inline TParams& SetBufLen(size_t buflen) noexcept {
-            BufLen = buflen;
-
-            return *this;
-        }
-
-        inline TParams& SetDict(const TStringBuf dict) noexcept {
-            Dict = dict;
-
-            return *this;
-        }
-
-        IOutputStream* Out;
-        ZLib::StreamType Type;
-        size_t CompressionLevel;
-        size_t BufLen;
-        TStringBuf Dict;
-    };
-
-    inline TZLibCompress(const TParams& params) {
-        Init(params);
-    }
-
-    inline TZLibCompress(IOutputStream* out, ZLib::StreamType type) {
-        Init(TParams(out).SetType(type));
-    }
-
-    inline TZLibCompress(IOutputStream* out, ZLib::StreamType type, size_t compression_level) {
-        Init(TParams(out).SetType(type).SetCompressionLevel(compression_level));
-    }
-
-    inline TZLibCompress(IOutputStream* out, ZLib::StreamType type, size_t compression_level, size_t buflen) {
-        Init(TParams(out).SetType(type).SetCompressionLevel(compression_level).SetBufLen(buflen));
-    }
-
-    ~TZLibCompress() override;
-
-private:
-    void Init(const TParams& opts);
-
-    void DoWrite(const void* buf, size_t size) override;
-    void DoFlush() override;
-    void DoFinish() override;
-
-public:
-    class TImpl;
-
-    /** To allow inline constructors. */
-    struct TDestruct {
-        static void Destroy(TImpl* impl);
-    };
-
-    THolder<TImpl, TDestruct> Impl_;
-};
-
-/**
- * Buffered ZLib decompressing stream.
- *
- * Supports efficient `ReadLine` calls and similar "reading in small pieces"
- * usage patterns.
- */
-class TBufferedZLibDecompress: public TBuffered<TZLibDecompress> {
-public:
-    template <class T>
-    inline TBufferedZLibDecompress(T* in, ZLib::StreamType type = ZLib::Auto, size_t buf = 1 << 13)
-        : TBuffered<TZLibDecompress>(buf, in, type)
-    {
-    }
-
-    ~TBufferedZLibDecompress() override;
-};
-
-/** @} */