koffi 2.3.10-beta.3 → 2.3.11

package/src/core/libcc/libcc.hh

@@ -136,7 +136,7 @@ static_assert(sizeof(double) == 8, "This code base is not designed to support si
 #define RG_STRINGIFY(a) RG_STRINGIFY_(a)
 #define RG_CONCAT_(a, b) a ## b
 #define RG_CONCAT(a, b) RG_CONCAT_(a, b)
-#define RG_UNIQUE_NAME(prefix) RG_CONCAT(prefix, __COUNTER__)
+#define RG_UNIQUE_NAME(prefix) RG_CONCAT(prefix, __LINE__)
 #define RG_FORCE_EXPAND(x) x
 #define RG_IGNORE (void)!
 
@@ -597,7 +597,7 @@ public:
     }; \
     static ClassName RG_UNIQUE_NAME(init); \
     ClassName::ClassName()
-#define RG_INIT(Name) RG_INIT_(RG_UNIQUE_NAME(InitHelper))
+#define RG_INIT(Name) RG_INIT_(RG_CONCAT(RG_UNIQUE_NAME(InitHelper), Name))
 
 #define RG_EXIT_(ClassName) \
     class ClassName { \
@@ -606,7 +606,7 @@ public:
     }; \
     static ClassName RG_UNIQUE_NAME(exit); \
     ClassName::~ClassName()
-#define RG_EXIT(Name) RG_EXIT_(RG_UNIQUE_NAME(ExitHelper))
+#define RG_EXIT(Name) RG_EXIT_(RG_CONCAT(RG_UNIQUE_NAME(ExitHelper), Name))
 
 template <typename T>
 T MultiCmp()
@@ -3791,13 +3791,22 @@ enum class CompressionType {
     None,
     Zlib,
     Gzip,
-    Brotli
+    Brotli,
+    LZ4
 };
 static const char *const CompressionTypeNames[] = {
     "None",
     "Zlib",
     "Gzip",
-    "Brotli"
+    "Brotli",
+    "LZ4"
+};
+static const char *const CompressionTypeExtensions[] = {
+    nullptr,
+    nullptr,
+    ".gz",
+    ".br",
+    ".lz4"
 };
 
 Span<const char> GetPathDirectory(Span<const char> filename);
@@ -3842,6 +3851,9 @@ struct FileInfo {
     int64_t mtime;
     int64_t btime;
     unsigned int mode;
+
+    uint32_t uid;
+    uint32_t gid;
 };
 
 enum class StatResult {
@@ -4202,6 +4214,9 @@ enum class CompressionSpeed {
     Fast
 };
 
+class StreamDecompressor;
+class StreamCompressor;
+
 class StreamReader {
     RG_DELETE_COPY(StreamReader)
 
@@ -4235,13 +4250,8 @@ class StreamReader {
         bool eof = false;
     } source;
 
-    struct {
-        CompressionType type = CompressionType::None;
-        union {
-            struct MinizInflateContext *miniz;
-            struct BrotliDecompressContext *brotli;
-        } u;
-    } compression;
+    CompressionType compression_type = CompressionType::None;
+    StreamDecompressor *decompressor = nullptr;
 
     int64_t raw_len = -1;
     Size raw_read = 0;
@@ -4278,7 +4288,7 @@ public:
     bool Rewind();
 
     const char *GetFileName() const { return filename; }
-    CompressionType GetCompressionType() const { return compression.type; }
+    CompressionType GetCompressionType() const { return compression_type; }
     bool IsValid() const { return filename && !error; }
     bool IsEOF() const { return eof; }
 
@@ -4301,10 +4311,9 @@ private:
 
     bool InitDecompressor(CompressionType type);
 
-    Size ReadInflate(Size max_len, void *out_buf);
-    Size ReadBrotli(Size max_len, void *out_buf);
-
     Size ReadRaw(Size max_len, void *out_buf);
+
+    friend class StreamDecompressor;
 };
 
 static inline Size ReadFile(const char *filename, CompressionType compression_type, Span<uint8_t> out_buf)
@@ -4351,6 +4360,42 @@ static inline Size ReadFile(const char *filename, Size max_len, HeapArray<char>
     return st.ReadAll(max_len, out_buf);
 }
 
+class StreamDecompressor {
+protected:
+    StreamReader *reader;
+
+public:
+    StreamDecompressor(StreamReader *reader) : reader(reader) {}
+    virtual ~StreamDecompressor() {}
+
+    virtual bool Init(CompressionType type) = 0;
+    virtual Size Read(Size max_len, void *out_buf) = 0;
+
+protected:
+    const char *GetFileName() const { return reader->filename; }
+    bool IsValid() const { return reader->IsValid(); }
+    bool IsSourceEOF() const { return reader->source.eof; }
+
+    Size ReadRaw(Size max_len, void *out_buf) { return reader->ReadRaw(max_len, out_buf); }
+
+    void SetEOF(bool eof) { reader->eof = eof; }
+};
+
+typedef StreamDecompressor *CreateDecompressorFunc(StreamReader *reader);
+
+class StreamDecompressorHelper {
+public:
+    StreamDecompressorHelper(CompressionType type, CreateDecompressorFunc *func);
+};
+
+#define RG_REGISTER_DECOMPRESSOR(Type, Cls) \
+    static StreamDecompressor *RG_UNIQUE_NAME(CreateDecompressor)(StreamReader *reader) \
+    { \
+        StreamDecompressor *decompressor = new Cls(reader); \
+        return decompressor; \
+    } \
+    static StreamDecompressorHelper RG_UNIQUE_NAME(CreateDecompressorHelper)((Type), RG_UNIQUE_NAME(CreateDecompressor))
+
 class LineReader {
     RG_DELETE_COPY(LineReader)
 
@@ -4420,14 +4465,9 @@ class StreamWriter {
         bool vt100;
     } dest;
 
-    struct {
-        CompressionType type = CompressionType::None;
-        CompressionSpeed speed = CompressionSpeed::Default;
-        union {
-            struct MinizDeflateContext *miniz;
-            struct BrotliEncoderStateStruct *brotli;
-        } u;
-    } compression;
+    CompressionType compression_type = CompressionType::None;
+    CompressionSpeed compression_speed = CompressionSpeed::Default;
+    StreamCompressor *compressor = nullptr;
 
     int64_t raw_written = 0;
 
@@ -4471,7 +4511,8 @@ public:
     bool Flush();
 
     const char *GetFileName() const { return filename; }
-    CompressionType GetCompressionType() const { return compression.type; }
+    CompressionType GetCompressionType() const { return compression_type; }
+    CompressionSpeed GetCompressionSpeed() const { return compression_speed; }
     bool IsVt100() const { return dest.vt100; }
     bool IsValid() const { return filename && !error; }
 
@@ -4490,10 +4531,9 @@ private:
 
     bool InitCompressor(CompressionType type, CompressionSpeed speed);
 
-    bool WriteDeflate(Span<const uint8_t> buf);
-    bool WriteBrotli(Span<const uint8_t> buf);
-
     bool WriteRaw(Span<const uint8_t> buf);
+
+    friend class StreamCompressor;
 };
 
 static inline bool WriteFile(Span<const uint8_t> buf, const char *filename, unsigned int flags = 0,
@@ -4511,6 +4551,40 @@ static inline bool WriteFile(Span<const char> buf, const char *filename, unsigne
     return st.Close();
 }
 
+class StreamCompressor {
+protected:
+    StreamWriter *writer;
+
+public:
+    StreamCompressor(StreamWriter *writer) : writer(writer) {}
+    virtual ~StreamCompressor() {}
+
+    virtual bool Init(CompressionType type, CompressionSpeed speed) = 0;
+    virtual bool Write(Span<const uint8_t> buf) = 0;
+    virtual bool Finalize() = 0;
+
+protected:
+    const char *GetFileName() const { return writer->filename; }
+    bool IsValid() const { return writer->IsValid(); }
+
+    bool WriteRaw(Span<const uint8_t> buf) { return writer->WriteRaw(buf); }
+};
+
+typedef StreamCompressor *CreateCompressorFunc(StreamWriter *writer);
+
+class StreamCompressorHelper {
+public:
+    StreamCompressorHelper(CompressionType type, CreateCompressorFunc *func);
+};
+
+#define RG_REGISTER_COMPRESSOR(Type, Cls) \
+    static StreamCompressor *RG_UNIQUE_NAME(CreateCompressor)(StreamWriter *writer) \
+    { \
+        StreamCompressor *compressor = new Cls(writer); \
+        return compressor; \
+    } \
+    static StreamCompressorHelper RG_UNIQUE_NAME(CreateCompressorHelper)((Type), RG_UNIQUE_NAME(CreateCompressor))
+
 bool SpliceStream(StreamReader *reader, int64_t max_len, StreamWriter *writer);
 
 // For convenience, don't close them

package/src/core/libcc/lz4.cc

@@ -0,0 +1,204 @@
+// Copyright 2023 Niels Martignène <niels.martignene@protonmail.com>
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the “Software”), to deal in 
+// the Software without restriction, including without limitation the rights to use,
+// copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+// Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+// OTHER DEALINGS IN THE SOFTWARE.
+
+#include "libcc.hh"
+
+#include "vendor/lz4/lib/lz4.h"
+#include "vendor/lz4/lib/lz4hc.h"
+#include "vendor/lz4/lib/lz4frame.h"
+
+namespace RG {
+
+class LZ4Decompressor: public StreamDecompressor {
+    LZ4F_dctx *decoder = nullptr;
+    bool done = false;
+
+    uint8_t in_buf[256 * 1024];
+    Size in_len = 0;
+    Size in_hint = RG_SIZE(in_buf);
+
+    uint8_t out_buf[256 * 1024];
+    Size out_len = 0;
+
+public:
+    LZ4Decompressor(StreamReader *reader) : StreamDecompressor(reader) {}
+    ~LZ4Decompressor();
+
+    bool Init(CompressionType type) override;
+    Size Read(Size max_len, void *out_buf) override;
+};
+
+LZ4Decompressor::~LZ4Decompressor()
+{
+    LZ4F_freeDecompressionContext(decoder);
+}
+
+bool LZ4Decompressor::Init(CompressionType)
+{
+    LZ4F_errorCode_t err = LZ4F_createDecompressionContext(&decoder, LZ4F_VERSION);
+    if (LZ4F_isError(err))
+        throw std::bad_alloc();
+
+    return true;
+}
+
+Size LZ4Decompressor::Read(Size max_len, void *user_buf)
+{
+    for (;;) {
+        if (out_len || done) {
+            Size copy_len = std::min(max_len, out_len);
+
+            out_len -= copy_len;
+            memcpy(user_buf, out_buf, copy_len);
+            memmove(out_buf, out_buf + copy_len, out_len);
+
+            SetEOF(!out_len && done);
+            return copy_len;
+        }
+
+        if (in_len < in_hint) {
+            Size raw_len = ReadRaw(in_hint - in_len, in_buf + in_len);
+            if (raw_len < 0)
+                return -1;
+            in_len += raw_len;
+        }
+
+        const uint8_t *next_in = in_buf;
+        uint8_t *next_out = out_buf + out_len;
+        size_t avail_in = (size_t)in_len;
+        size_t avail_out = (size_t)(RG_SIZE(out_buf) - out_len);
+
+        LZ4F_decompressOptions_t opt = {};
+        size_t ret = LZ4F_decompress(decoder, next_out, &avail_out, next_in, &avail_in, &opt);
+
+        if (!ret) {
+            done = true;
+        } else if (LZ4F_isError(ret)) {
+            LogError("Malformed LZ4 stream in '%1': %2", GetFileName(), LZ4F_getErrorName(ret));
+            return -1;
+        }
+
+        memmove_safe(in_buf, in_buf + avail_in, (size_t)in_len - avail_in);
+        in_len -= avail_in;
+        in_hint = std::min(RG_SIZE(in_buf), (Size)ret);
+
+        out_len += (Size)avail_out;
+    }
+
+    RG_UNREACHABLE();
+}
+
+class LZ4Compressor: public StreamCompressor {
+    LZ4F_cctx *encoder = nullptr;
+    LZ4F_preferences_t prefs = {};
+
+    HeapArray<uint8_t> dynamic_buf;
+
+public:
+    LZ4Compressor(StreamWriter *writer) : StreamCompressor(writer) {}
+    ~LZ4Compressor();
+
+    bool Init(CompressionType type, CompressionSpeed speed) override;
+    bool Write(Span<const uint8_t> buf) override;
+    bool Finalize() override;
+};
+
+LZ4Compressor::~LZ4Compressor()
+{
+    LZ4F_freeCompressionContext(encoder);
+}
+
+bool LZ4Compressor::Init(CompressionType, CompressionSpeed speed)
+{
+    LZ4F_errorCode_t err = LZ4F_createCompressionContext(&encoder, LZ4F_VERSION);
+    if (LZ4F_isError(err))
+        throw std::bad_alloc();
+
+    switch (speed) {
+        case CompressionSpeed::Default: { prefs.compressionLevel = LZ4HC_CLEVEL_MIN; } break;
+        case CompressionSpeed::Slow: { prefs.compressionLevel = LZ4HC_CLEVEL_MAX; } break;
+        case CompressionSpeed::Fast: { prefs.compressionLevel = 0; } break;
+    }
+
+    dynamic_buf.Grow(LZ4F_HEADER_SIZE_MAX);
+
+    size_t ret = LZ4F_compressBegin(encoder, dynamic_buf.end(), dynamic_buf.capacity - dynamic_buf.len, &prefs);
+
+    if (LZ4F_isError(ret)) {
+        LogError("Failed to start LZ4 stream for '%1': %2", GetFileName(), LZ4F_getErrorName(ret));
+        return false;
+    }
+
+    dynamic_buf.len += ret;
+
+    return true;
+}
+
+bool LZ4Compressor::Write(Span<const uint8_t> buf)
+{
+    size_t needed = LZ4F_compressBound((size_t)buf.len, &prefs);
+    dynamic_buf.Grow((Size)needed);
+
+    size_t ret = LZ4F_compressUpdate(encoder, dynamic_buf.end(), (size_t)(dynamic_buf.capacity - dynamic_buf.len),
+                                     buf.ptr, (size_t)buf.len, nullptr);
+
+    if (LZ4F_isError(ret)) {
+        LogError("Failed to write LZ4 stream for '%1': %2", GetFileName(), LZ4F_getErrorName(ret));
+        return false;
+    }
+
+    dynamic_buf.len += (Size)ret;
+
+    if (dynamic_buf.len >= 512) {
+        if (!WriteRaw(dynamic_buf))
+            return false;
+        dynamic_buf.len = 0;
+    }
+
+    return true;
+}
+
+bool LZ4Compressor::Finalize()
+{
+    size_t needed = LZ4F_compressBound(0, &prefs);
+    dynamic_buf.Grow((Size)needed);
+
+    size_t ret = LZ4F_compressEnd(encoder, dynamic_buf.end(),
+                                  (size_t)(dynamic_buf.capacity - dynamic_buf.len), nullptr);
+
+    if (LZ4F_isError(ret)) {
+        LogError("Failed to finalize LZ4 stream for '%1': %2", GetFileName(), LZ4F_getErrorName(ret));
+        return false;
+    }
+
+    dynamic_buf.len += (Size)ret;
+
+    if (!WriteRaw(dynamic_buf))
+        return false;
+    dynamic_buf.len = 0;
+
+    return true;
+}
+
+RG_REGISTER_DECOMPRESSOR(CompressionType::LZ4, LZ4Decompressor);
+RG_REGISTER_COMPRESSOR(CompressionType::LZ4, LZ4Compressor);
+
+}