catboost 1.25.1 → 1.26.0

package/inc/util/system/direct_io.h

@@ -1,71 +0,0 @@
-#pragma once
-
-#include "align.h"
-
-#include "file.h"
-#include <util/generic/buffer.h>
-
-// Supports Linux Direct-IO:
-// - Simple buffering logic.
-// - Default buffer size of 128KB matches VM page writeback granularity, to maximize IO throughput.
-// - Supports writing odd sized files by turning off direct IO for the last chunk.
-class TDirectIOBufferedFile {
-public:
-    TDirectIOBufferedFile(const TString& path, EOpenMode oMode, size_t buflen = 1 << 17);
-    ~TDirectIOBufferedFile();
-
-    void FlushData();
-    void Finish();
-    size_t Read(void* buffer, size_t byteCount);
-    void Write(const void* buffer, size_t byteCount);
-    size_t Pread(void* buffer, size_t byteCount, ui64 offset);
-    void Pwrite(const void* buffer, size_t byteCount, ui64 offset);
-
-    inline bool IsOpen() const {
-        return true;
-    }
-
-    inline ui64 GetWritePosition() const {
-        return WritePosition;
-    }
-
-    inline ui64 GetLength() const {
-        return FlushedBytes + DataLen;
-    }
-
-    inline FHANDLE GetHandle() {
-        return File.GetHandle();
-    }
-
-    inline void FallocateNoResize(ui64 length) {
-        File.FallocateNoResize(length);
-    }
-
-private:
-    inline bool IsAligned(i64 value) {
-        return Alignment ? value == AlignDown<i64>(value, Alignment) : true;
-    }
-
-    inline bool IsAligned(const void* value) {
-        return Alignment ? value == AlignDown(value, Alignment) : true;
-    }
-
-    size_t PreadSafe(void* buffer, size_t byteCount, ui64 offset);
-    size_t ReadFromFile(void* buffer, size_t byteCount, ui64 offset);
-    void WriteToFile(const void* buf, size_t len, ui64 position);
-    void WriteToBuffer(const void* buf, size_t len, ui64 position);
-    void SetDirectIO(bool value);
-
-private:
-    TFile File;
-    size_t Alignment;
-    size_t BufLen;
-    size_t DataLen;
-    void* Buffer;
-    TBuffer BufferStorage;
-    ui64 ReadPosition;
-    ui64 WritePosition;
-    ui64 FlushedBytes;
-    ui64 FlushedToDisk;
-    bool DirectIO;
-};

package/inc/util/system/dynlib.h

@@ -1,119 +0,0 @@
-#pragma once
-
-#include "defaults.h"
-
-#include <util/generic/ptr.h>
-#include <util/generic/string.h>
-
-#define Y_GET_FUNC(dll, name) FUNC_##name((dll).Sym(#name))
-#define Y_GET_FUNC_OPTIONAL(dll, name) FUNC_##name((dll).SymOptional(#name))
-
-#ifdef _win32_
-#define DEFAULT_DLLOPEN_FLAGS 0
-#else
-#include <dlfcn.h>
-
-#ifndef RTLD_GLOBAL
-#define RTLD_GLOBAL (0)
-#endif
-
-#define DEFAULT_DLLOPEN_FLAGS (RTLD_NOW | RTLD_GLOBAL)
-#endif
-
-class TDynamicLibrary {
-public:
-    TDynamicLibrary() noexcept;
-    TDynamicLibrary(const TString& path, int flags = DEFAULT_DLLOPEN_FLAGS);
-    ~TDynamicLibrary();
-
-    void Open(const char* path, int flags = DEFAULT_DLLOPEN_FLAGS);
-    void Close() noexcept;
-    void* SymOptional(const char* name) noexcept;
-    void* Sym(const char* name);
-    bool IsLoaded() const noexcept;
-    void SetUnloadable(bool unloadable); // Set to false to avoid unloading on destructor
-
-private:
-    class TImpl;
-    THolder<TImpl> Impl_;
-};
-
-// a wrapper for a symbol
-template <class TLib>
-class TExternalSymbol {
-private:
-    TLib* PLib;
-    TDynamicLibrary* DLib;
-    TString lname;
-    TString vname;
-
-public:
-    TExternalSymbol() noexcept {
-        PLib = nullptr;
-        DLib = nullptr;
-    }
-    TExternalSymbol(const TExternalSymbol& es) {
-        PLib = nullptr;
-        DLib = nullptr;
-        if (es.IsDynamic())
-            Open(es.LibName().data(), es.VtblName().data());
-        else if (es.IsStatic())
-            SetSym(es.Symbol());
-    }
-    TExternalSymbol& operator=(const TExternalSymbol& es) {
-        if (this != &es) {
-            Close();
-            if (es.IsDynamic())
-                Open(es.LibName().data(), es.VtblName().data());
-            else if (es.IsStatic())
-                SetSym(es.Symbol());
-        }
-        return *this;
-    }
-    ~TExternalSymbol() {
-        delete DLib;
-    }
-    // set the symbol from dynamic source
-    void Open(const char* lib_name, const char* vtbl_name) {
-        if (DLib != nullptr || PLib != nullptr)
-            return;
-        try {
-            DLib = new TDynamicLibrary();
-            DLib->Open(lib_name);
-            PLib = (TLib*)DLib->Sym(vtbl_name);
-        } catch (...) {
-            delete DLib;
-            DLib = nullptr;
-            throw;
-        }
-        lname = lib_name;
-        vname = vtbl_name;
-    }
-    // set the symbol from static source
-    void SetSym(TLib* pl) noexcept {
-        if (DLib == nullptr && PLib == nullptr)
-            PLib = pl;
-    }
-    void Close() noexcept {
-        delete DLib;
-        DLib = 0;
-        PLib = 0;
-        lname.remove();
-        vname.remove();
-    }
-    TLib* Symbol() const noexcept {
-        return PLib;
-    }
-    const TString& LibName() const noexcept {
-        return lname;
-    }
-    const TString& VtblName() const noexcept {
-        return vname;
-    }
-    bool IsStatic() const noexcept {
-        return DLib == nullptr && PLib != nullptr;
-    }
-    bool IsDynamic() const noexcept {
-        return DLib && DLib->IsLoaded() && PLib != nullptr;
-    }
-};

package/inc/util/system/env.h

@@ -1,32 +0,0 @@
-#pragma once
-
-#include <util/generic/string.h>
-
-/**
- * Search the environment list provided by the host environment for associated variable.
- *
- * @param key   String identifying the name of the environmental variable to look for
- * @param def   String that returns if environmental variable not found by key
- *
- * @return      String that is associated with the matched environment variable or empty string if
- *              such variable is missing.
- *
- * @note        Use it only in pair with `SetEnv` as there may be inconsistency in their behaviour
- *              otherwise.
- * @note        Calls to `GetEnv` and `SetEnv` from different threads must be synchronized.
- * @see         SetEnv
- */
-TString GetEnv(const TString& key, const TString& def = TString());
-
-/**
- * Add or change environment variable provided by the host environment.
- *
- * @key         String identifying the name of the environment variable to set or change
- * @value       Value to assign
-
- * @note        Use it only in pair with `GetEnv` as there may be inconsistency in their behaviour
- *              otherwise.
- * @note        Calls to `GetEnv` and `SetEnv` from different threads must be synchronized.
- * @see         GetEnv
- */
-void SetEnv(const TString& key, const TString& value);

package/inc/util/system/error.h

@@ -1,95 +0,0 @@
-#pragma once
-
-#include "defaults.h"
-
-#if defined(_win_)
-#include <winerror.h>
-#include <errno.h>
-
-#undef E_FAIL
-#undef ERROR_TIMEOUT
-
-#if defined(_MSC_VER)
-#undef EADDRINUSE
-#undef EADDRNOTAVAIL
-#undef EAFNOSUPPORT
-#undef EALREADY
-#undef ECANCELED
-#undef ECONNABORTED
-#undef ECONNREFUSED
-#undef ECONNRESET
-#undef EDESTADDRREQ
-#undef EHOSTUNREACH
-#undef EINPROGRESS
-#undef EISCONN
-#undef ELOOP
-#undef EMSGSIZE
-#undef ENETDOWN
-#undef ENETRESET
-#undef ENETUNREACH
-#undef ENOBUFS
-#undef ENOPROTOOPT
-#undef ENOTCONN
-#undef ENOTSOCK
-#undef EOPNOTSUPP
-#undef EPROTONOSUPPORT
-#undef EPROTOTYPE
-#undef ETIMEDOUT
-#undef EWOULDBLOCK
-#undef ENAMETOOLONG
-#undef ENOTEMPTY
-
-#define EWOULDBLOCK WSAEWOULDBLOCK
-#define EINPROGRESS WSAEINPROGRESS
-#define EALREADY WSAEALREADY
-#define ENOTSOCK WSAENOTSOCK
-#define EDESTADDRREQ WSAEDESTADDRREQ
-#define EMSGSIZE WSAEMSGSIZE
-#define EPROTOTYPE WSAEPROTOTYPE
-#define ENOPROTOOPT WSAENOPROTOOPT
-#define EPROTONOSUPPORT WSAEPROTONOSUPPORT
-#define ESOCKTNOSUPPORT WSAESOCKTNOSUPPORT
-#define EOPNOTSUPP WSAEOPNOTSUPP
-#define EPFNOSUPPORT WSAEPFNOSUPPORT
-#define EAFNOSUPPORT WSAEAFNOSUPPORT
-#define EADDRINUSE WSAEADDRINUSE
-#define EADDRNOTAVAIL WSAEADDRNOTAVAIL
-#define ENETDOWN WSAENETDOWN
-#define ENETUNREACH WSAENETUNREACH
-#define ENETRESET WSAENETRESET
-#define ECONNABORTED WSAECONNABORTED
-#define ECONNRESET WSAECONNRESET
-#define ENOBUFS WSAENOBUFS
-#define EISCONN WSAEISCONN
-#define ENOTCONN WSAENOTCONN
-#define ESHUTDOWN WSAESHUTDOWN
-#define ETOOMANYREFS WSAETOOMANYREFS
-#define ETIMEDOUT WSAETIMEDOUT
-#define ECONNREFUSED WSAECONNREFUSED
-#define ELOOP WSAELOOP
-#define ENAMETOOLONG WSAENAMETOOLONG
-#define EHOSTDOWN WSAEHOSTDOWN
-#define EHOSTUNREACH WSAEHOSTUNREACH
-#define ENOTEMPTY WSAENOTEMPTY
-#define EPROCLIM WSAEPROCLIM
-#define EUSERS WSAEUSERS
-#define ESTALE WSAESTALE
-#define EREMOTE WSAEREMOTE
-#define ECANCELED WSAECANCELLED
-#endif
-
-#define EDQUOT WSAEDQUOT
-#endif
-
-void ClearLastSystemError();
-int LastSystemError();
-void LastSystemErrorText(char* str, size_t size, int code);
-const char* LastSystemErrorText(int code);
-
-inline const char* LastSystemErrorText() {
-    return LastSystemErrorText(LastSystemError());
-}
-
-inline void LastSystemErrorText(char* str, size_t size) {
-    LastSystemErrorText(str, size, LastSystemError());
-}

package/inc/util/system/event.h

@@ -1,122 +0,0 @@
-#pragma once
-
-#include <util/generic/ptr.h>
-#include <util/datetime/base.h>
-
-struct TEventResetType {
-    enum ResetMode {
-        rAuto,   // the state will be nonsignaled after Wait() returns
-        rManual, // we need call Reset() to set the state to nonsignaled.
-    };
-};
-
-/**
- * DEPRECATED!
- *
- * Use TAutoEvent, TManualEvent for the direct replacement.
- * Use TManualEvent to prevent SEGFAULT (http://nga.at.yandex-team.ru/5772).
- */
-class TSystemEvent: public TEventResetType {
-public:
-    TSystemEvent(ResetMode rmode = rManual);
-    TSystemEvent(const TSystemEvent& other) noexcept;
-    TSystemEvent& operator=(const TSystemEvent& other) noexcept;
-
-    ~TSystemEvent();
-
-    void Reset() noexcept;
-    void Signal() noexcept;
-
-    /*
-     * return true if signaled, false if timed out.
-     */
-    bool WaitD(TInstant deadLine) noexcept;
-
-    /*
-     * return true if signaled, false if timed out.
-     */
-    inline bool WaitT(TDuration timeOut) noexcept {
-        return WaitD(timeOut.ToDeadLine());
-    }
-
-    /*
-     * wait infinite time
-     */
-    inline void WaitI() noexcept {
-        WaitD(TInstant::Max());
-    }
-
-    //return true if signaled, false if timed out.
-    inline bool Wait(ui32 timer) noexcept {
-        return WaitT(TDuration::MilliSeconds(timer));
-    }
-
-    inline bool Wait() noexcept {
-        WaitI();
-
-        return true;
-    }
-
-private:
-    class TEvImpl;
-    TIntrusivePtr<TEvImpl> EvImpl_;
-};
-
-class TAutoEvent: public TSystemEvent {
-public:
-    TAutoEvent()
-        : TSystemEvent(TSystemEvent::rAuto)
-    {
-    }
-
-private:
-    void Reset() noexcept;
-};
-
-/**
- * Prevents from a "shortcut problem" (see http://nga.at.yandex-team.ru/5772): if Wait will be called after Signaled
- * flag set to true in Signal method but before CondVar.BroadCast - Wait will shortcut (without actual wait on condvar).
- * If Wait thread will destruct event - Signal thread will do broadcast on a destructed CondVar.
- */
-class TManualEvent {
-public:
-    TManualEvent()
-        : Ev(TEventResetType::rManual)
-    {
-    }
-
-    void Reset() noexcept {
-        TSystemEvent{Ev}.Reset();
-    }
-
-    void Signal() noexcept {
-        TSystemEvent{Ev}.Signal();
-    }
-
-    /** return true if signaled, false if timed out. */
-    bool WaitD(TInstant deadLine) noexcept {
-        return TSystemEvent{Ev}.WaitD(deadLine);
-    }
-
-    /** return true if signaled, false if timed out. */
-    inline bool WaitT(TDuration timeOut) noexcept {
-        return TSystemEvent{Ev}.WaitT(timeOut);
-    }
-
-    /** Wait infinite time */
-    inline void WaitI() noexcept {
-        TSystemEvent{Ev}.WaitI();
-    }
-
-    /** return true if signaled, false if timed out. */
-    inline bool Wait(ui32 timer) noexcept {
-        return TSystemEvent{Ev}.Wait(timer);
-    }
-
-    inline bool Wait() noexcept {
-        return TSystemEvent{Ev}.Wait();
-    }
-
-private:
-    TSystemEvent Ev;
-};

package/inc/util/system/execpath.h

@@ -1,17 +0,0 @@
-#pragma once
-
-#include <util/generic/fwd.h>
-
-// NOTE: This function has rare sporadic failures (throws exceptions) on FreeBSD. See REVIEW:54297
-const TString& GetExecPath();
-
-/**
- * Get openable path to the binary currently being executed.
- *
- * The path does not match the original binary location, but stays openable even
- * if the binary was moved or removed.
- *
- * On UNIX variants, utilizes the /proc FS. On Windows, equivalent to
- * GetExecPath.
- */
-const TString& GetPersistentExecPath();

package/inc/util/system/fasttime.h

@@ -1,6 +0,0 @@
-#pragma once
-
-#include "datetime.h"
-
-/// Fast but possibly less accurate microseconds since epoch
-ui64 InterpolatedMicroSeconds();

package/inc/util/system/fhandle.h

@@ -1,27 +0,0 @@
-#pragma once
-
-#include "defaults.h"
-
-using WIN_HANDLE = void*;
-#define INVALID_WIN_HANDLE ((WIN_HANDLE)(long)-1)
-
-using UNIX_HANDLE = int;
-#define INVALID_UNIX_HANDLE -1
-
-#if defined(_win_)
-using FHANDLE = WIN_HANDLE;
-#define INVALID_FHANDLE INVALID_WIN_HANDLE
-#elif defined(_unix_)
-using FHANDLE = UNIX_HANDLE;
-#define INVALID_FHANDLE INVALID_UNIX_HANDLE
-#else
-#error
-#endif
-
-#if defined(_cygwin_)
-using OS_HANDLE = WIN_HANDLE;
-#define INVALID_OS_HANDLE INVALID_WIN_HANDLE
-#else
-using OS_HANDLE = FHANDLE;
-#define INVALID_OS_HANDLE INVALID_FHANDLE
-#endif

package/inc/util/system/file.h

@@ -1,210 +0,0 @@
-#pragma once
-
-#include "fhandle.h"
-#include "flock.h"
-
-#include <util/generic/flags.h>
-#include <util/generic/ptr.h>
-#include <util/generic/noncopyable.h>
-
-#include <cstdio>
-
-enum EOpenModeFlag {
-    OpenExisting = 0,  // Opens a file. It fails if the file does not exist.
-    TruncExisting = 1, // Opens a file and truncates it to zero size. It fails if the file does not exist.
-    OpenAlways = 2,    // Opens a file, always. If a file does not exist, it creates a file.
-    CreateNew = 3,     // Creates a new file. It fails if a specified file exists.
-    CreateAlways = 4,  // Creates a new file, always. If a file exists, it overwrites the file.
-    MaskCreation = 7,
-
-    RdOnly = 8,  // open for reading only
-    WrOnly = 16, // open for writing only
-    RdWr = 24,   // open for reading and writing
-    MaskRW = 24,
-
-    Seq = 0x20,             // file access is primarily sequential (POSIX_FADV_SEQUENTIAL)
-    Direct = 0x40,          // file is being opened with no system caching (Does not work as intended! See implementation)
-    Temp = 0x80,            // avoid writing data back to disk if sufficient cache memory is available (no op for linux)
-    ForAppend = 0x100,      // write appends data to the end of file (O_APPEND)
-    Transient = 0x200,      // actually, temporary file - 'delete on close' for windows, unlink after creation for unix
-    NoReuse = 0x400,        // no second access expected (POSIX_FADV_NOREUSE)
-    CloseOnExec = 0x800,    // set close-on-exec right at open (O_CLOEXEC)
-    DirectAligned = 0x1000, // file is actually being opened with no system caching (may require buffer alignment) (O_DIRECT)
-    Sync = 0x2000,          // no write call will return before the data is transferred to the disk (O_SYNC)
-    NoReadAhead = 0x4000,   // no sequential access expected, opposite for Seq (POSIX_FADV_RANDOM)
-
-    AXOther = 0x00010000,
-    AWOther = 0x00020000,
-    AROther = 0x00040000,
-    AXGroup = 0x00100000,
-    AWGroup = 0x00200000,
-    ARGroup = 0x00400000,
-    AXUser = 0x01000000,
-    AWUser = 0x02000000,
-    ARUser = 0x04000000,
-    AX = AXUser | AXGroup | AXOther,
-    AW = AWUser | AWGroup,
-    AR = ARUser | ARGroup | AROther,
-    ARW = AR | AW,
-    AMask = 0x0FFF0000,
-};
-
-Y_DECLARE_FLAGS(EOpenMode, EOpenModeFlag)
-Y_DECLARE_OPERATORS_FOR_FLAGS(EOpenMode)
-
-TString DecodeOpenMode(ui32 openMode);
-
-enum SeekDir {
-    sSet = 0,
-    sCur = 1,
-    sEnd = 2,
-};
-
-class TFileHandle: public TNonCopyable {
-public:
-    constexpr TFileHandle() = default;
-
-    /// Warning: takes ownership of fd, so closes it in destructor.
-    inline TFileHandle(FHANDLE fd) noexcept
-        : Fd_(fd)
-    {
-    }
-
-    inline TFileHandle(TFileHandle&& other) noexcept
-        : Fd_(other.Fd_)
-    {
-        other.Fd_ = INVALID_FHANDLE;
-    }
-
-    TFileHandle(const TString& fName, EOpenMode oMode) noexcept;
-
-    inline ~TFileHandle() {
-        Close();
-    }
-
-    bool Close() noexcept;
-
-    inline FHANDLE Release() noexcept {
-        FHANDLE ret = Fd_;
-        Fd_ = INVALID_FHANDLE;
-        return ret;
-    }
-
-    inline void Swap(TFileHandle& r) noexcept {
-        DoSwap(Fd_, r.Fd_);
-    }
-
-    inline operator FHANDLE() const noexcept {
-        return Fd_;
-    }
-
-    inline bool IsOpen() const noexcept {
-        return Fd_ != INVALID_FHANDLE;
-    }
-
-    i64 GetPosition() const noexcept;
-    i64 GetLength() const noexcept;
-
-    i64 Seek(i64 offset, SeekDir origin) noexcept;
-    bool Resize(i64 length) noexcept;
-    bool Reserve(i64 length) noexcept;
-    bool FallocateNoResize(i64 length) noexcept;
-    bool Flush() noexcept;
-    //flush data only, without file metadata
-    bool FlushData() noexcept;
-    i32 Read(void* buffer, ui32 byteCount) noexcept;
-    i32 Write(const void* buffer, ui32 byteCount) noexcept;
-    i32 Pread(void* buffer, ui32 byteCount, i64 offset) const noexcept;
-    i32 Pwrite(const void* buffer, ui32 byteCount, i64 offset) const noexcept;
-    int Flock(int op) noexcept;
-
-    FHANDLE Duplicate() const noexcept;
-    int Duplicate2Posix(int dstHandle) const noexcept;
-
-    //dup2 - like semantics, return true on success
-    bool LinkTo(const TFileHandle& fh) const noexcept;
-
-    //very low-level methods
-    bool SetDirect();
-    void ResetDirect();
-
-    /* Manual file cache management, length = 0 means "as much as possible" */
-
-    //measure amount of cached data in bytes, returns -1 if failed
-    i64 CountCache(i64 offset = 0, i64 length = 0) const noexcept;
-    //read data into cache and optionally wait for completion
-    void PrefetchCache(i64 offset = 0, i64 length = 0, bool wait = true) const noexcept;
-    //remove clean and unused data from cache
-    void EvictCache(i64 offset = 0, i64 length = 0) const noexcept;
-    //flush unwritten data in this range and optionally wait for completion
-    bool FlushCache(i64 offset = 0, i64 length = 0, bool wait = true) noexcept;
-
-private:
-    FHANDLE Fd_ = INVALID_FHANDLE;
-};
-
-class TFile {
-public:
-    TFile();
-    /// Takes ownership of handle, so closes it when the last holder of descriptor dies.
-    explicit TFile(FHANDLE fd);
-    TFile(FHANDLE fd, const TString& fname);
-    TFile(const TString& fName, EOpenMode oMode);
-    ~TFile();
-
-    void Close();
-
-    const TString& GetName() const noexcept;
-    i64 GetPosition() const noexcept;
-    i64 GetLength() const noexcept;
-    bool IsOpen() const noexcept;
-    FHANDLE GetHandle() const noexcept;
-
-    i64 Seek(i64 offset, SeekDir origin);
-    void Resize(i64 length);
-    void Reserve(i64 length);
-    void FallocateNoResize(i64 length);
-    void Flush();
-    void FlushData();
-
-    void LinkTo(const TFile& f) const;
-    TFile Duplicate() const;
-
-    size_t Read(void* buf, size_t len);
-    size_t ReadOrFail(void* buf, size_t len);
-    i32 RawRead(void* buf, size_t len);
-    void Load(void* buf, size_t len);
-    void Write(const void* buf, size_t len);
-    size_t Pread(void* buf, size_t len, i64 offset) const;
-    i32 RawPread(void* buf, ui32 len, i64 offset) const;
-    void Pload(void* buf, size_t len, i64 offset) const;
-    void Pwrite(const void* buf, size_t len, i64 offset) const;
-    void Flock(int op);
-
-    //do not use, their meaning very platform-dependant
-    void SetDirect();
-    void ResetDirect();
-
-    /* Manual file cache management, length = 0 means "as much as possible" */
-
-    //measure amount of cached data in bytes, returns -1 if failed
-    i64 CountCache(i64 offset = 0, i64 length = 0) const noexcept;
-    //read data into cache and optionally wait for completion
-    void PrefetchCache(i64 offset = 0, i64 length = 0, bool wait = true) const noexcept;
-    //remove clean and unused data from cache, incomplete pages could stay
-    void EvictCache(i64 offset = 0, i64 length = 0) const noexcept;
-    //flush unwritten data in this range and optionally wait for completion
-    void FlushCache(i64 offset = 0, i64 length = 0, bool wait = true);
-
-    static TFile Temporary(const TString& prefix);
-    static TFile ForAppend(const TString& path);
-
-private:
-    class TImpl;
-    TSimpleIntrusivePtr<TImpl> Impl_;
-};
-
-TFile Duplicate(FILE*);
-TFile Duplicate(int);
-
-bool PosixDisableReadAhead(FHANDLE fileHandle, void* addr) noexcept;