yaz0 0.3.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8497431199299dfc86cb43d084da005a9ed7a11f83b7c3a01fc62a8dfb8ca0c
-  data.tar.gz: 9636cbce864ed5d1f7e3e639531cb69a5e329ae97e10aefcb1ef2c2d506901c5
+  metadata.gz: 857c2a0a5b230d48ad3e295e5877096d6287dd1c6bb1f072817a83e5a00b6851
+  data.tar.gz: 87bdadd4138b0f29d95061dc82073373c7cff344b00f0dd2b40cc581f2264c71
 SHA512:
-  metadata.gz: 73536d99b19fc8a56b5171cf031b3ae897ce9058f84d8874931bd5155b594b190f8f47795d47c506b3022091c78522744112bb5293e563517434f5b285d4e9e4
-  data.tar.gz: e986e0043ee983ed7c3301acb176bfdf37a9a5dd6ce22ceacba73a3479973c851ec841c17cbc69f8a6d909ee5afaa70a9d95c8c646ef1eae0f35c0df3610cde6
+  metadata.gz: 40aab200a225dbc5131a96500dd08011f5054fc8916525caf6fcfa7fed7a3344d559de56f0a36875bc9785773d1be9c9301bd899821e58f447f4fbfc155f699a
+  data.tar.gz: 29aa243dd3c0111414cd33a600cacc0dbc47c3dc159199ab7539813949c5b4fb8dd7726c6bd95c92c8f377a8bf68bb7354e73dc0a3c873258ced29b7797e1aa9

data/Gemfile

@@ -3,6 +3,6 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in yaz0.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem "rake", "~> 13.0"
 gem "rake-compiler"
 gem "rspec", "~> 3.0"

data/ext/yaz0/ext_yaz0.c

@@ -0,0 +1,165 @@
+#include "ext_yaz0.h"
+#include <stdio.h>
+#include <stdint.h>
+
+#define BUFSIZE 0x4000
+
+static void ext_yaz0_stream_free(void*);
+
+static VALUE class_yaz0_stream;
+static VALUE class_yaz0_error;
+static VALUE class_yaz0_error_bad_magic;
+static VALUE class_yaz0_error_end_of_file;
+
+static struct rb_data_type_struct type_yaz0_stream = {
+    "yaz0_stream",
+    { NULL, ext_yaz0_stream_free, NULL },
+    NULL,
+    NULL,
+    RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static void
+ext_yaz0_stream_free(void* s)
+{
+    yaz0Destroy((Yaz0Stream*)s);
+}
+
+static VALUE
+ext_yaz0_stream_alloc(VALUE klass)
+{
+    Yaz0Stream* s;
+    int ret;
+
+    ret = yaz0Init(&s);
+    if (ret == YAZ0_OUT_OF_MEMORY)
+        rb_raise(rb_eNoMemError, "Out of memory");
+    return TypedData_Wrap_Struct(klass, &type_yaz0_stream, s);
+}
+
+static void*
+run_NoGVL(void* arg)
+{
+    return (void*)(intptr_t)yaz0Run((Yaz0Stream*)arg);
+}
+
+static VALUE
+run(VALUE self, VALUE io_in, VALUE io_out, int compress, int size, int level)
+{
+    Yaz0Stream* s;
+    VALUE buffer_in;
+    VALUE buffer_out;
+    VALUE tmp;
+    int in_is_str;
+    int ret;
+
+    in_is_str = TYPE(io_in) == T_STRING;
+
+    TypedData_Get_Struct(self, Yaz0Stream, &type_yaz0_stream, s);
+    if (compress)
+        yaz0ModeCompress(s, size, level);
+    else
+        yaz0ModeDecompress(s);
+
+    /* Init the buffers */
+    if (!in_is_str)
+    {
+        buffer_in = rb_str_new(NULL, 0);
+        rb_str_resize(buffer_in, BUFSIZE);
+        rb_gc_register_address(&buffer_in);
+        rb_funcall(io_in, rb_intern("read"), 2, INT2FIX(BUFSIZE), buffer_in);
+    }
+    else
+    {
+        buffer_in = rb_obj_dup(io_in);
+        rb_gc_register_address(&buffer_in);
+    }
+    yaz0Input(s, RSTRING_PTR(buffer_in), (uint32_t)RSTRING_LEN(buffer_in));
+
+    buffer_out = rb_str_new(NULL, 0);
+    rb_str_resize(buffer_out, BUFSIZE);
+    rb_gc_register_address(&buffer_out);
+    yaz0Output(s, RSTRING_PTR(buffer_out), BUFSIZE);
+
+    for (;;)
+    {
+        ret = (int)(intptr_t)rb_thread_call_without_gvl(run_NoGVL, s, RUBY_UBF_IO, NULL);
+        switch (ret)
+        {
+        case YAZ0_NEED_AVAIL_IN:
+            /* Need more input */
+            if (in_is_str)
+            {
+                rb_gc_unregister_address(&buffer_in);
+                rb_gc_unregister_address(&buffer_out);
+                rb_raise(class_yaz0_error_end_of_file, "Unexpected end of file");
+            }
+
+            tmp = rb_funcall(io_in, rb_intern("read"), 2, INT2FIX(BUFSIZE), buffer_in);
+            if (tmp == Qnil)
+            {
+                rb_gc_unregister_address(&buffer_in);
+                rb_gc_unregister_address(&buffer_out);
+                rb_raise(class_yaz0_error_end_of_file, "Unexpected end of file");
+            }
+            yaz0Input(s, RSTRING_PTR(buffer_in), (uint32_t)RSTRING_LEN(buffer_in));
+            break;
+        case YAZ0_NEED_AVAIL_OUT:
+            /* Need more output */
+            rb_str_resize(buffer_out, yaz0OutputChunkSize(s));
+            rb_funcall(io_out, rb_intern("write"), 1, buffer_out);
+            rb_str_resize(buffer_out, BUFSIZE);
+            yaz0Output(s, RSTRING_PTR(buffer_out), BUFSIZE);
+            break;
+        case YAZ0_BAD_MAGIC:
+            rb_gc_unregister_address(&buffer_in);
+            rb_gc_unregister_address(&buffer_out);
+            rb_raise(class_yaz0_error_bad_magic, "Bad magic");
+            break;
+        case YAZ0_OK:
+            goto end;
+        }
+    }
+
+end:
+    /* There might still be unflushed output */
+    rb_str_resize(buffer_out, yaz0OutputChunkSize(s));
+    rb_funcall(io_out, rb_intern("write"), 1, buffer_out);
+
+    rb_gc_unregister_address(&buffer_in);
+    rb_gc_unregister_address(&buffer_out);
+
+    return Qnil;
+}
+
+static VALUE
+ext_yaz0_stream_raw_decompress(VALUE self, VALUE io_in, VALUE io_out)
+{
+    return run(self, io_in, io_out, 0, 0, 0);
+}
+
+static VALUE
+ext_yaz0_stream_raw_compress(VALUE self, VALUE io_in, VALUE io_out, VALUE size, VALUE level)
+{
+    Check_Type(size, T_FIXNUM);
+    Check_Type(level, T_FIXNUM);
+    return run(self, io_in, io_out, 1, FIX2INT(size), FIX2INT(level));
+}
+
+void
+Init_yaz0(void)
+{
+    VALUE mod;
+    mod = rb_define_module("Yaz0");
+
+    /* Error classes */
+    class_yaz0_error = rb_define_class_under(mod, "Error", rb_eStandardError);
+    class_yaz0_error_bad_magic = rb_define_class_under(mod, "BadMagicError", class_yaz0_error);
+    class_yaz0_error_end_of_file = rb_define_class_under(mod, "EndOfFileError", class_yaz0_error);
+
+    /* Stream */
+    class_yaz0_stream = rb_define_class_under(mod, "Stream", rb_cObject);
+    rb_define_alloc_func(class_yaz0_stream, ext_yaz0_stream_alloc);
+    rb_define_method(class_yaz0_stream, "raw_decompress", ext_yaz0_stream_raw_decompress, 2);
+    rb_define_method(class_yaz0_stream, "raw_compress", ext_yaz0_stream_raw_compress, 4);
+}

data/ext/yaz0/ext_yaz0.h

@@ -0,0 +1,8 @@
+#ifndef EXT_YAZ0_H
+#define EXT_YAZ0_H 1
+
+#include <ruby.h>
+#include <ruby/thread.h>
+#include <yaz0.h>
+
+#endif /* YAZ0_H */

data/ext/yaz0/extconf.rb

@@ -1,3 +1,13 @@
 require "mkmf"
 
-create_makefile("yaz0/yaz0")
+extension_name = 'yaz0/yaz0'
+dir_config(extension_name)
+
+libyaz0_src = Dir[File.join(__dir__, "../../libyaz0/src/libyaz0/**/*.c")].map{|x| File.expand_path(x)}
+
+$srcs = libyaz0_src + ["ext_yaz0.c"]
+
+$VPATH << File.expand_path(File.join(__dir__, "../../libyaz0/src/libyaz0"))
+$INCFLAGS << " -I#{File.expand_path(File.join(__dir__, "../../libyaz0/include"))}"
+
+create_makefile(extension_name)

data/lib/yaz0/version.rb

@@ -1,3 +1,3 @@
 module Yaz0
-  VERSION = "0.3.0"
+  VERSION = "0.4.2"
 end

data/lib/yaz0.rb

@@ -1,5 +1,47 @@
+require "stringio"
 require "yaz0/version"
 require "yaz0/yaz0"
 
 module Yaz0
+  class Stream
+    def decompress(src, dst = nil)
+      if dst
+        raw_decompress(src, dst)
+        nil
+      else
+        dst = StringIO.new
+        raw_decompress(src, dst)
+        dst.string
+      end
+    end
+
+    def compress(src, dst_or_opts = nil, opts = {})
+      if dst_or_opts.is_a?(Hash)
+        opts = dst_or_opts
+        dst = nil
+      else
+        dst = dst_or_opts
+      end
+
+      level = opts[:level] || 6
+      size = opts[:size] || src.size
+
+      if dst
+        raw_compress(src, dst, size, level)
+        nil
+      else
+        dst = StringIO.new
+        raw_compress(src, dst, size, level)
+        dst.string
+      end
+    end
+  end
+
+  def self.decompress(*args)
+    Yaz0::Stream.new.decompress(*args)
+  end
+
+  def self.compress(*args)
+    Yaz0::Stream.new.compress(*args)
+  end
 end

data/libyaz0/include/yaz0.h

@@ -0,0 +1,34 @@
+#ifndef YAZ0_H
+#define YAZ0_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#if defined(__cplusplus)
+# define YAZ0_API extern "C"
+#else
+# define YAZ0_API
+#endif
+
+#define YAZ0_OK             0
+#define YAZ0_NEED_AVAIL_IN  1
+#define YAZ0_NEED_AVAIL_OUT 2
+#define YAZ0_BAD_MAGIC      (-1)
+#define YAZ0_OUT_OF_MEMORY  (-2)
+
+#define YAZ0_DEFAULT_LEVEL  6
+
+typedef struct Yaz0Stream Yaz0Stream;
+
+YAZ0_API int yaz0Init(Yaz0Stream** stream);
+YAZ0_API int yaz0Destroy(Yaz0Stream* stream);
+YAZ0_API int yaz0ModeDecompress(Yaz0Stream* stream);
+YAZ0_API int yaz0ModeCompress(Yaz0Stream* stream, uint32_t size, int level);
+YAZ0_API int yaz0Run(Yaz0Stream* stream);
+YAZ0_API int yaz0Input(Yaz0Stream* stream, const void* data, uint32_t size);
+YAZ0_API int yaz0Output(Yaz0Stream* stream, void* data, uint32_t size);
+
+YAZ0_API uint32_t yaz0OutputChunkSize(const Yaz0Stream* stream);
+YAZ0_API uint32_t yaz0DecompressedSize(const Yaz0Stream* stream);
+
+#endif /* YAZ0_H */

data/libyaz0/src/libyaz0/CMakeLists.txt

@@ -0,0 +1,3 @@
+file(GLOB SOURCES "*.c")
+add_library(libyaz0 STATIC ${SOURCES})
+set_target_properties(libyaz0 PROPERTIES OUTPUT_NAME "yaz0")

data/libyaz0/src/libyaz0/compress.c

@@ -0,0 +1,392 @@
+#include <string.h>
+#include <stdio.h>
+#include "libyaz0.h"
+
+static const uint32_t kProbesPerLevel[] = {
+    0x0,
+    0x1,
+    0x2,
+    0x4,
+    0x8,
+    0x10,
+    0x40,
+    0x100,
+    0x200,
+    0x1000
+};
+
+static uint32_t hash(uint8_t a, uint8_t b, uint8_t c)
+{
+    uint32_t x = (uint32_t)a | ((uint32_t)b << 8) | ((uint32_t)c << 16);
+    x = ((x >> 16) ^ x) * 0x45d9f3b;
+    x = (x >> 16) ^ x;
+    return x;
+}
+
+static void hashWrite(Yaz0Stream* s, uint32_t h, uint32_t offset)
+{
+    uint32_t maxProbes;
+    uint32_t bucket;
+    uint32_t tmpBucket;
+    uint32_t oldest;
+    uint32_t entry;
+    int32_t pos;
+
+    oldest = 0xffffffff;
+    maxProbes = kProbesPerLevel[s->level];
+    for (uint32_t i = 0; i < maxProbes; ++i)
+    {
+        tmpBucket = (h + i) % HASH_MAX_ENTRIES;
+        entry = s->htEntries[tmpBucket];
+        if (entry == 0xffffffff)
+        {
+            s->htSize++;
+            bucket = tmpBucket;
+            break;
+        }
+        pos = (int32_t)(s->totalOut - entry);
+        if (pos > 0x1000)
+        {
+            bucket = tmpBucket;
+            break;
+        }
+        if (entry < oldest)
+        {
+            oldest = entry;
+            bucket = tmpBucket;
+        }
+    }
+    s->htEntries[bucket] = s->totalOut + offset;
+    s->htHashes[bucket] = h;
+}
+
+static void rebuildHashTable(Yaz0Stream* s)
+{
+    uint32_t newEntries[HASH_MAX_ENTRIES];
+    uint32_t newHashes[HASH_MAX_ENTRIES];
+    uint32_t entry;
+    uint32_t pos;
+    uint32_t h;
+    uint32_t bucket;
+    uint32_t size;
+
+    memset(newEntries, 0xff, sizeof(newEntries));
+    memset(newHashes, 0xff, sizeof(newHashes));
+    size = 0;
+
+    for (uint32_t i = 0; i < HASH_MAX_ENTRIES; ++i)
+    {
+        entry = s->htEntries[i];
+        if (entry == 0xffffffff)
+            continue;
+        pos = s->totalOut - entry;
+        if (pos > 0x1000)
+            continue;
+
+        /* Entry still good - move to the table */
+        h = s->htHashes[i];
+        bucket = h % HASH_MAX_ENTRIES;
+        for (;;)
+        {
+            if (newEntries[bucket] == 0xffffffff)
+            {
+                newEntries[bucket] = entry;
+                newHashes[bucket] = h;
+                size++;
+                break;
+            }
+            bucket = (bucket + 1) % HASH_MAX_ENTRIES;
+        }
+    }
+
+    s->htSize = size;
+    memcpy(s->htEntries, newEntries, sizeof(newEntries));
+    memcpy(s->htHashes, newHashes, sizeof(newHashes));
+}
+
+static uint32_t maxSize(Yaz0Stream* stream)
+{
+    /* the extra byte is for look-aheads */
+    static const uint32_t maxNecessary = 0x888 + 1;
+    uint32_t max;
+
+    max = stream->decompSize - stream->totalOut;
+    if (max > maxNecessary)
+        max = maxNecessary;
+    return max;
+}
+
+/* start: start of avail data */
+/* end: end of avail data */
+/* We need to write more data at the end of the window */
+
+static int feed(Yaz0Stream* s)
+{
+    uint32_t avail;
+    uint32_t min;
+    uint32_t max;
+    uint32_t size;
+    int ret;
+
+    /* Check how much data we have */
+    if (s->window_start > s->window_end)
+        avail = WINDOW_SIZE - s->window_start + s->window_end;
+    else
+        avail = s->window_end - s->window_start;
+    if (avail >= maxSize(s))
+        return YAZ0_OK;
+
+    /* We need more data */
+    min = maxSize(s) - avail;
+    max = WINDOW_SIZE - 0x1000 - avail;
+    if (max > s->sizeIn - s->cursorIn)
+        max = s->sizeIn - s->cursorIn;
+    if (max < min)
+        ret = YAZ0_NEED_AVAIL_IN;
+    else
+        ret = YAZ0_OK;
+    if (s->window_end + max >= WINDOW_SIZE)
+    {
+        /* We might need two copies */
+        size = WINDOW_SIZE - s->window_end;
+        memcpy(s->window + s->window_end, s->in + s->cursorIn, size);
+        s->cursorIn += size;
+        s->window_end = 0;
+        max -= size;
+    }
+    memcpy(s->window + s->window_end, s->in + s->cursorIn, max);
+    s->cursorIn += max;
+    s->window_end += max;
+    return ret;
+}
+
+static uint32_t matchSize(Yaz0Stream* s, uint32_t offset, uint32_t pos, uint32_t hintSize)
+{
+    uint32_t size = 0;
+    uint32_t start = s->window_start + offset;
+    uint32_t cursorA = (start + WINDOW_SIZE - pos) % WINDOW_SIZE;
+    uint32_t cursorB = start % WINDOW_SIZE;
+    uint32_t maxSize;
+
+    maxSize = s->decompSize - s->totalOut;
+    if (maxSize > 0x111)
+        maxSize = 0x111;
+    if (hintSize)
+    {
+        if (s->window[(cursorA + hintSize) % WINDOW_SIZE] != s->window[(cursorB + hintSize) % WINDOW_SIZE])
+            return 0;
+    }
+    for (;;)
+    {
+        if (s->window[(cursorA + size) % WINDOW_SIZE] != s->window[(cursorB + size) % WINDOW_SIZE])
+            break;
+        size++;
+        if (size == maxSize)
+            break;
+    }
+    return size;
+}
+
+static void findHashMatch(Yaz0Stream* s, uint32_t h, uint32_t offset, uint32_t* outSize, uint32_t* outPos)
+{
+    uint32_t bucket;
+    uint32_t entry;
+    uint32_t bestSize;
+    uint32_t bestPos;
+    uint32_t size;
+    uint32_t pos;
+    uint32_t maxProbes;
+
+    bestSize = 0;
+    bestPos = 0;
+    maxProbes = kProbesPerLevel[s->level];
+    for (uint32_t i = 0; i < maxProbes; ++i)
+    {
+        bucket = (h + i) % HASH_MAX_ENTRIES;
+        entry = s->htEntries[bucket];
+        if (entry == 0xffffffff)
+            break;
+        if (s->htHashes[bucket] == h)
+        {
+            pos = s->totalOut + offset - entry;
+            if (pos > 0x1000)
+                continue;
+            size = matchSize(s, offset, pos, bestSize);
+            if (size > bestSize)
+            {
+                bestSize = size;
+                bestPos = pos;
+            }
+        }
+    }
+
+    if (bestSize < 3)
+    {
+        *outSize = 0;
+    }
+    else
+    {
+        *outSize = bestSize;
+        *outPos = bestPos;
+    }
+}
+
+static void emitGroup(Yaz0Stream* s, int count, const uint32_t* arrSize, const uint32_t* arrPos)
+{
+    uint8_t header;
+    uint32_t size;
+    uint32_t pos;
+
+    header = 0;
+    for (int i = 0; i < count; ++i)
+    {
+        if (!arrSize[i])
+            header |= (1 << (7 - i));
+    }
+    s->out[s->cursorOut++] = header;
+    for (int i = 0; i < count; ++i)
+    {
+        size = arrSize[i];
+        pos = arrPos[i];
+        if (!size)
+            s->out[s->cursorOut++] = (uint8_t)pos;
+        else
+        {
+            pos--;
+            if (size >= 0x12)
+            {
+                /* 3 bytes */
+                s->out[s->cursorOut++] = (uint8_t)(pos >> 8);
+                s->out[s->cursorOut++] = (uint8_t)pos;
+                s->out[s->cursorOut++] = (uint8_t)(size - 0x12);
+            }
+            else
+            {
+                /* 2 bytes */
+                s->out[s->cursorOut++] = (uint8_t)(pos >> 8) | (uint8_t)((size - 2) << 4);
+                s->out[s->cursorOut++] = (uint8_t)pos;
+            }
+        }
+    }
+}
+
+static void compressGroup(Yaz0Stream* s)
+{
+    int groupCount;
+    uint32_t h;
+    uint32_t size;
+    uint32_t pos;
+    uint32_t nextSize;
+    uint32_t nextPos;
+    uint32_t arrSize[8];
+    uint32_t arrPos[8];
+    uint8_t a;
+    uint8_t b;
+    uint8_t c;
+    uint8_t d;
+
+    for (groupCount = 0; groupCount < 8; ++groupCount)
+    {
+        a = s->window[s->window_start];
+        b = s->window[(s->window_start + 1) % WINDOW_SIZE];
+        c = s->window[(s->window_start + 2) % WINDOW_SIZE];
+        d = s->window[(s->window_start + 3) % WINDOW_SIZE];
+        h = hash(a, b, c);
+        findHashMatch(s, h, 0, &size, &pos);
+        hashWrite(s, h, 0);
+
+        h = hash(b, c, d);
+        findHashMatch(s, h, 1, &nextSize, &nextPos);
+
+        if (!size || nextSize > size)
+        {
+            arrSize[groupCount] = 0;
+            arrPos[groupCount] = s->window[s->window_start];
+            s->window_start += 1;
+            s->totalOut += 1;
+        }
+        else
+        {
+            arrSize[groupCount] = size;
+            arrPos[groupCount] = pos;
+            for (uint32_t i = 1; i < size; ++i)
+            {
+                a = b;
+                b = c;
+                c = s->window[(s->window_start + 2 + i) % WINDOW_SIZE];
+                h = hash(a, b, c);
+                hashWrite(s, h, i);
+            }
+            s->window_start += size;
+            s->totalOut += size;
+        }
+        s->window_start %= WINDOW_SIZE;
+        if (s->totalOut >= s->decompSize)
+        {
+            groupCount++;
+            break;
+        }
+    }
+    if (s->htSize > HASH_REBUILD)
+        rebuildHashTable(s);
+    emitGroup(s, groupCount, arrSize, arrPos);
+}
+
+int yaz0ModeCompress(Yaz0Stream* s, uint32_t size, int level)
+{
+    memset(s, 0, sizeof(*s));
+    s->mode = MODE_COMPRESS;
+    s->decompSize = size;
+    if (level < 1)
+        level = 1;
+    else if (level > 9)
+        level = 9;
+    s->level = level;
+    for (int i = 0; i < HASH_MAX_ENTRIES; ++i)
+    {
+        s->htHashes[i]  = 0xffffffff;
+        s->htEntries[i] = 0xffffffff;
+    }
+    return YAZ0_OK;
+}
+
+int yaz0_RunCompress(Yaz0Stream* stream)
+{
+    uint32_t tmp;
+    int ret;
+
+    /* Write headers */
+    if (!stream->headersDone)
+    {
+        if (stream->sizeOut < 16)
+            return YAZ0_NEED_AVAIL_OUT;
+        memcpy(stream->out, "Yaz0", 4);
+        tmp = swap32(stream->decompSize);
+        memcpy(stream->out + 4, &tmp, 4);
+        tmp = 0;
+        memcpy(stream->out + 8, &tmp, 4);
+        memcpy(stream->out + 12, &tmp, 4);
+        stream->cursorOut += 16;
+        stream->headersDone = 1;
+    }
+
+    /* Compress */
+    for (;;)
+    {
+        /* Check EOF */
+        if (stream->totalOut >= stream->decompSize)
+            return YAZ0_OK;
+
+        /* Check output space */
+        if (stream->sizeOut - stream->cursorOut < 1 + 8 * 3)
+            return YAZ0_NEED_AVAIL_OUT;
+
+        /* Check that we have consumed enough input */
+        ret = feed(stream);
+        if (ret)
+            return ret;
+
+        /* Compress one chunk */
+        compressGroup(stream);
+    }
+}