yaz0 0.3.0 → 0.4.0

data/libyaz0/src/libyaz0/compress.c

@@ -0,0 +1,434 @@
+#include <string.h>
+#include <stdio.h>
+#include "libyaz0.h"
+
+static const uint32_t kProbesPerLevel[] = {
+    0x0,
+    0x1,
+    0x2,
+    0x4,
+    0x8,
+    0x10,
+    0x40,
+    0x100,
+    0x200,
+    0x1000
+};
+
+static uint32_t hash(uint8_t a, uint8_t b, uint8_t c)
+{
+    uint32_t x = (uint32_t)a | ((uint32_t)b << 8) | ((uint32_t)c << 16);
+    x = ((x >> 16) ^ x) * 0x45d9f3b;
+    x = (x >> 16) ^ x;
+    return x;
+}
+
+static void hashWrite(Yaz0Stream* s, uint32_t h, uint32_t offset)
+{
+    uint32_t maxProbes;
+    uint32_t bucket;
+    uint32_t tmpBucket;
+    uint32_t oldest;
+    uint32_t entry;
+    int32_t pos;
+
+    oldest = 0xffffffff;
+    maxProbes = kProbesPerLevel[s->level];
+    for (uint32_t i = 0; i < maxProbes; ++i)
+    {
+        tmpBucket = (h + i) % HASH_MAX_ENTRIES;
+        entry = s->htEntries[tmpBucket];
+        if (entry == 0xffffffff)
+        {
+            s->htSize++;
+            bucket = tmpBucket;
+            break;
+        }
+        pos = (int32_t)(s->totalOut - entry);
+        if (pos > 0x1000)
+        {
+            bucket = tmpBucket;
+            break;
+        }
+        if (entry < oldest)
+        {
+            oldest = entry;
+            bucket = tmpBucket;
+        }
+    }
+    s->htEntries[bucket] = s->totalOut + offset;
+    s->htHashes[bucket] = h;
+}
+
+static void rebuildHashTable(Yaz0Stream* s)
+{
+    uint32_t newEntries[HASH_MAX_ENTRIES];
+    uint32_t newHashes[HASH_MAX_ENTRIES];
+    uint32_t entry;
+    uint32_t pos;
+    uint32_t h;
+    uint32_t bucket;
+    uint32_t size;
+
+    memset(newEntries, 0xff, sizeof(newEntries));
+    memset(newHashes, 0xff, sizeof(newHashes));
+    size = 0;
+
+    for (uint32_t i = 0; i < HASH_MAX_ENTRIES; ++i)
+    {
+        entry = s->htEntries[i];
+        if (entry == 0xffffffff)
+            continue;
+        pos = s->totalOut - entry;
+        if (pos > 0x1000)
+            continue;
+
+        /* Entry still good - move to the table */
+        h = s->htHashes[i];
+        bucket = h % HASH_MAX_ENTRIES;
+        for (;;)
+        {
+            if (newEntries[bucket] == 0xffffffff)
+            {
+                newEntries[bucket] = entry;
+                newHashes[bucket] = h;
+                size++;
+                break;
+            }
+            bucket = (bucket + 1) % HASH_MAX_ENTRIES;
+        }
+    }
+
+    s->htSize = size;
+    memcpy(s->htEntries, newEntries, sizeof(newEntries));
+    memcpy(s->htHashes, newHashes, sizeof(newHashes));
+}
+
+static uint32_t maxSize(Yaz0Stream* stream)
+{
+    /* the extra byte is for look-aheads */
+    static const uint32_t maxNecessary = 0x888 + 1;
+    uint32_t max;
+
+    max = stream->decompSize - stream->totalOut;
+    if (max > maxNecessary)
+        max = maxNecessary;
+    return max;
+}
+
+/* start: start of avail data */
+/* end: end of avail data */
+/* We need to write more data at the end of the window */
+
+static int feed(Yaz0Stream* s)
+{
+    uint32_t avail;
+    uint32_t min;
+    uint32_t max;
+    uint32_t size;
+    int ret;
+
+    /* Check how much data we have */
+    if (s->window_start > s->window_end)
+        avail = WINDOW_SIZE - s->window_start + s->window_end;
+    else
+        avail = s->window_end - s->window_start;
+    if (avail >= maxSize(s))
+        return YAZ0_OK;
+
+    /* We need more data */
+    min = maxSize(s) - avail;
+    max = WINDOW_SIZE - 0x1000 - avail;
+    if (max > s->sizeIn - s->cursorIn)
+        max = s->sizeIn - s->cursorIn;
+    if (max < min)
+        ret = YAZ0_NEED_AVAIL_IN;
+    else
+        ret = YAZ0_OK;
+    if (s->window_end + max >= WINDOW_SIZE)
+    {
+        /* We might need two copies */
+        size = WINDOW_SIZE - s->window_end;
+        memcpy(s->window + s->window_end, s->in + s->cursorIn, size);
+        s->cursorIn += size;
+        s->window_end = 0;
+        max -= size;
+    }
+    memcpy(s->window + s->window_end, s->in + s->cursorIn, max);
+    s->cursorIn += max;
+    s->window_end += max;
+    return ret;
+}
+
+static uint32_t matchSize(Yaz0Stream* s, uint32_t offset, uint32_t pos, uint32_t hintSize)
+{
+    uint32_t size = 0;
+    uint32_t start = s->window_start + offset;
+    uint32_t cursorA = (start + WINDOW_SIZE - pos) % WINDOW_SIZE;
+    uint32_t cursorB = start % WINDOW_SIZE;
+    uint32_t maxSize;
+
+    maxSize = s->decompSize - s->totalOut;
+    if (maxSize > 0x111)
+        maxSize = 0x111;
+    if (hintSize)
+    {
+        if (s->window[(cursorA + hintSize) % WINDOW_SIZE] != s->window[(cursorB + hintSize) % WINDOW_SIZE])
+            return 0;
+    }
+    for (;;)
+    {
+        if (s->window[(cursorA + size) % WINDOW_SIZE] != s->window[(cursorB + size) % WINDOW_SIZE])
+            break;
+        size++;
+        if (size == maxSize)
+            break;
+    }
+    return size;
+}
+
+static void findHashMatch(Yaz0Stream* s, uint32_t h, uint32_t offset, uint32_t* outSize, uint32_t* outPos)
+{
+    uint32_t bucket;
+    uint32_t entry;
+    uint32_t bestSize;
+    uint32_t bestPos;
+    uint32_t size;
+    uint32_t pos;
+    uint32_t maxProbes;
+
+    bestSize = 0;
+    bestPos = 0;
+    maxProbes = kProbesPerLevel[s->level];
+    for (uint32_t i = 0; i < maxProbes; ++i)
+    {
+        bucket = (h + i) % HASH_MAX_ENTRIES;
+        entry = s->htEntries[bucket];
+        if (entry == 0xffffffff)
+            break;
+        if (s->htHashes[bucket] == h)
+        {
+            pos = s->totalOut + offset - entry;
+            if (pos > 0x1000)
+                continue;
+            size = matchSize(s, offset, pos, bestSize);
+            if (size > bestSize)
+            {
+                bestSize = size;
+                bestPos = pos;
+            }
+        }
+    }
+
+    if (bestSize < 3)
+    {
+        *outSize = 0;
+    }
+    else
+    {
+        *outSize = bestSize;
+        *outPos = bestPos;
+    }
+}
+
+static void findEarlyZeroes(Yaz0Stream* s, uint32_t* outSize, uint32_t* outPos)
+{
+    uint32_t size;
+    uint32_t bestSize;
+    uint32_t bestPos;
+
+    bestSize = 0;
+    bestPos = 0;
+    for (uint32_t pos = 0x1000; pos > 0x990; --pos)
+    {
+        size = matchSize(s, 0, pos, 0);
+        if (size > bestSize)
+        {
+            bestSize = size;
+            bestPos = pos;
+        }
+    }
+
+    if (bestSize < 3)
+    {
+        *outSize = 0;
+    }
+    else
+    {
+        *outSize = bestSize;
+        *outPos = bestPos;
+    }
+}
+
+static void emitGroup(Yaz0Stream* s, int count, const uint32_t* arrSize, const uint32_t* arrPos)
+{
+    uint8_t header;
+    uint32_t size;
+    uint32_t pos;
+
+    header = 0;
+    for (int i = 0; i < count; ++i)
+    {
+        if (!arrSize[i])
+            header |= (1 << (7 - i));
+    }
+    s->out[s->cursorOut++] = header;
+    for (int i = 0; i < count; ++i)
+    {
+        size = arrSize[i];
+        pos = arrPos[i];
+        if (!size)
+            s->out[s->cursorOut++] = (uint8_t)pos;
+        else
+        {
+            pos--;
+            if (size >= 0x12)
+            {
+                /* 3 bytes */
+                s->out[s->cursorOut++] = (uint8_t)(pos >> 8);
+                s->out[s->cursorOut++] = (uint8_t)pos;
+                s->out[s->cursorOut++] = (uint8_t)(size - 0x12);
+            }
+            else
+            {
+                /* 2 bytes */
+                s->out[s->cursorOut++] = (uint8_t)(pos >> 8) | (uint8_t)((size - 2) << 4);
+                s->out[s->cursorOut++] = (uint8_t)pos;
+            }
+        }
+    }
+}
+
+static void compressGroup(Yaz0Stream* s)
+{
+    int groupCount;
+    uint32_t h;
+    uint32_t size;
+    uint32_t pos;
+    uint32_t nextSize;
+    uint32_t nextPos;
+    uint32_t arrSize[8];
+    uint32_t arrPos[8];
+    uint8_t a;
+    uint8_t b;
+    uint8_t c;
+    uint8_t d;
+
+    for (groupCount = 0; groupCount < 8; ++groupCount)
+    {
+        a = s->window[s->window_start];
+        b = s->window[(s->window_start + 1) % WINDOW_SIZE];
+        c = s->window[(s->window_start + 2) % WINDOW_SIZE];
+        d = s->window[(s->window_start + 3) % WINDOW_SIZE];
+        h = hash(a, b, c);
+        findHashMatch(s, h, 0, &size, &pos);
+        hashWrite(s, h, 0);
+
+        /* Check for early zero */
+        if (s->window[s->window_start] == 0 && s->totalOut < 0x1000)
+        {
+            findEarlyZeroes(s, &nextSize, &nextPos);
+            if (nextSize > size)
+            {
+                size = nextSize;
+                pos = nextPos;
+            }
+        }
+
+        nextSize = 0;
+        nextPos = 0;
+        h = hash(b, c, d);
+        findHashMatch(s, h, 1, &nextSize, &nextPos);
+
+        if (!size || nextSize > size)
+        {
+            arrSize[groupCount] = 0;
+            arrPos[groupCount] = s->window[s->window_start];
+            s->window_start += 1;
+            s->totalOut += 1;
+        }
+        else
+        {
+            arrSize[groupCount] = size;
+            arrPos[groupCount] = pos;
+            for (uint32_t i = 1; i < size; ++i)
+            {
+                a = b;
+                b = c;
+                c = s->window[(s->window_start + 2 + i) % WINDOW_SIZE];
+                h = hash(a, b, c);
+                hashWrite(s, h, i);
+            }
+            s->window_start += size;
+            s->totalOut += size;
+        }
+        s->window_start %= WINDOW_SIZE;
+        if (s->totalOut >= s->decompSize)
+        {
+            groupCount++;
+            break;
+        }
+    }
+    if (s->htSize > HASH_REBUILD)
+        rebuildHashTable(s);
+    emitGroup(s, groupCount, arrSize, arrPos);
+}
+
+int yaz0ModeCompress(Yaz0Stream* s, uint32_t size, int level)
+{
+    memset(s, 0, sizeof(*s));
+    s->mode = MODE_COMPRESS;
+    s->decompSize = size;
+    if (level < 1)
+        level = 1;
+    else if (level > 9)
+        level = 9;
+    s->level = level;
+    for (int i = 0; i < HASH_MAX_ENTRIES; ++i)
+    {
+        s->htHashes[i]  = 0xffffffff;
+        s->htEntries[i] = 0xffffffff;
+    }
+    return YAZ0_OK;
+}
+
+int yaz0_RunCompress(Yaz0Stream* stream)
+{
+    uint32_t tmp;
+    int ret;
+
+    /* Write headers */
+    if (!stream->headersDone)
+    {
+        if (stream->sizeOut < 16)
+            return YAZ0_NEED_AVAIL_OUT;
+        memcpy(stream->out, "Yaz0", 4);
+        tmp = swap32(stream->decompSize);
+        memcpy(stream->out + 4, &tmp, 4);
+        tmp = 0;
+        memcpy(stream->out + 8, &tmp, 4);
+        memcpy(stream->out + 12, &tmp, 4);
+        stream->cursorOut += 16;
+        stream->headersDone = 1;
+    }
+
+    /* Compress */
+    for (;;)
+    {
+        /* Check EOF */
+        if (stream->totalOut >= stream->decompSize)
+            return YAZ0_OK;
+
+        /* Check output space */
+        if (stream->sizeOut - stream->cursorOut < 1 + 8 * 3)
+            return YAZ0_NEED_AVAIL_OUT;
+
+        /* Check that we have consumed enough input */
+        ret = feed(stream);
+        if (ret)
+            return ret;
+
+        /* Compress one chunk */
+        compressGroup(stream);
+    }
+}

data/libyaz0/src/libyaz0/decompress.c

@@ -0,0 +1,197 @@
+#include <string.h>
+#include <stdio.h>
+#include "libyaz0.h"
+
+int yaz0ModeDecompress(Yaz0Stream* s)
+{
+    memset(s, 0, sizeof(*s));
+    s->mode = MODE_DECOMPRESS;
+    return YAZ0_OK;
+}
+
+void loadAux(Yaz0Stream* stream, uint32_t size)
+{
+    if (stream->sizeIn - stream->cursorIn < size)
+        size = stream->sizeIn - stream->cursorIn;
+    for (uint32_t i = 0; i < size; i++)
+        stream->auxBuf[stream->auxSize++] = stream->in[stream->cursorIn++];
+}
+
+int flush(Yaz0Stream* stream)
+{
+    size_t chunkSize;
+    size_t outSize;
+
+    if (stream->window_start == stream->window_end)
+        return YAZ0_OK;
+    if (stream->cursorOut >= stream->sizeOut)
+        return YAZ0_NEED_AVAIL_OUT;
+    outSize = stream->sizeOut - stream->cursorOut;
+    if (stream->window_start > stream->window_end)
+    {
+        /* Wrap around - we will need 2 copies */
+        chunkSize = WINDOW_SIZE - stream->window_start;
+        if (chunkSize > outSize)
+            chunkSize = outSize;
+        memcpy(stream->out + stream->cursorOut, stream->window + stream->window_start, chunkSize);
+        stream->cursorOut += chunkSize;
+        stream->window_start += chunkSize;
+        stream->window_start %= WINDOW_SIZE;
+        outSize -= chunkSize;
+    }
+    /* Copy the rest */
+    chunkSize = stream->window_end - stream->window_start;
+    if (chunkSize > outSize)
+        chunkSize = outSize;
+    memcpy(stream->out + stream->cursorOut, stream->window + stream->window_start, chunkSize);
+    stream->cursorOut += chunkSize;
+    stream->window_start += chunkSize;
+    return YAZ0_OK;
+}
+
+static int yaz0_ReadHeaders(Yaz0Stream* stream)
+{
+    loadAux(stream, 16 - stream->auxSize);
+    if (stream->auxSize < 16)
+        return YAZ0_NEED_AVAIL_IN;
+    if (memcmp(stream->auxBuf, "Yaz0", 4))
+        return YAZ0_BAD_MAGIC;
+    stream->decompSize = swap32(*(uint32_t*)&stream->auxBuf[4]);
+    stream->auxSize = 0;
+    return YAZ0_OK;
+}
+
+static uint32_t windowFreeSize(Yaz0Stream* stream)
+{
+    uint32_t size;
+    if (stream->window_start > stream->window_end)
+        size = WINDOW_SIZE - stream->window_start + stream->window_end;
+    else
+        size = stream->window_end - stream->window_start;
+    return WINDOW_SIZE - size;
+}
+
+static int ensureWindowFree(Yaz0Stream* stream)
+{
+    static const uint32_t maxSize = 0x111 * 8;
+
+    if (windowFreeSize(stream) < maxSize)
+    {
+        flush(stream);
+        if (windowFreeSize(stream) < maxSize)
+            return YAZ0_NEED_AVAIL_OUT;
+    }
+    return YAZ0_OK;
+}
+
+int yaz0_DoDecompress(Yaz0Stream* stream)
+{
+    uint8_t     groupBit;
+    uint8_t     byte;
+    int         ret;
+    uint16_t    n;
+    uint16_t    r;
+    for (;;)
+    {
+        /* No group and no EOF - We need to read */
+        if (stream->groupCount == 0)
+        {
+            /* Before we read, we eant to ensure the window is large enough */
+            /* This will avoid a lot of checks and let us write the whole group */
+            ret = ensureWindowFree(stream);
+            if (ret)
+                return ret;
+            if (stream->cursorIn >= stream->sizeIn)
+                return YAZ0_NEED_AVAIL_IN;
+            stream->groupHeader = stream->in[stream->cursorIn++];
+            stream->groupCount = 8;
+        }
+
+        /* We have a group! */
+        while (stream->groupCount)
+        {
+            /* Get the group bit */
+            groupBit = stream->groupHeader & (1 << (stream->groupCount - 1));
+            if (groupBit)
+            {
+                /* Group bit is 1 - direct write */
+                if (stream->cursorIn >= stream->sizeIn)
+                    return YAZ0_NEED_AVAIL_IN;
+
+                /* Everything ok, copy the byte */
+                byte = stream->in[stream->cursorIn++];
+                stream->window[stream->window_end++] = byte;
+                stream->window_end %= WINDOW_SIZE;
+                stream->totalOut++;
+            }
+            else
+            {
+                if (stream->auxSize < 2)
+                {
+                    loadAux(stream, 2 - stream->auxSize);
+                    if (stream->auxSize < 2)
+                        return YAZ0_NEED_AVAIL_IN;
+                }
+                /* Do we need 2 or 3 bytes? */
+                n = ((uint8_t)stream->auxBuf[0]) >> 4;
+                if (!n)
+                {
+                    /* We have a large chunk */
+                    loadAux(stream, 1);
+                    if (stream->auxSize < 3)
+                        return YAZ0_NEED_AVAIL_IN;
+                    n = (uint8_t)stream->auxBuf[2] + 0x12;
+                }
+                else
+                {
+                    /* We have a small chunk */
+                    n = ((uint8_t)stream->auxBuf[0] >> 4) + 2;
+                }
+                r = ((uint16_t)(((uint8_t)stream->auxBuf[0] & 0x0f) << 8) | ((uint8_t)stream->auxBuf[1]));
+                r++;
+                /* Reset the aux buffer */
+                uint32_t cursor = (stream->window_end + WINDOW_SIZE - r) % WINDOW_SIZE;
+                for (int i = 0; i < n; ++i)
+                {
+                    stream->window[stream->window_end++] = stream->window[cursor++];
+                    stream->window_end %= WINDOW_SIZE;
+                    cursor %= WINDOW_SIZE;
+                }
+                stream->auxSize = 0;
+                stream->totalOut += n;
+            }
+            stream->groupCount--;
+            /* Check for EOF */
+            if (stream->totalOut >= stream->decompSize)
+                return YAZ0_OK;
+        }
+    }
+}
+
+int yaz0_RunDecompress(Yaz0Stream* stream)
+{
+    int ret;
+
+    /* Check the headers */
+    if (!stream->headersDone)
+    {
+        ret = yaz0_ReadHeaders(stream);
+        if (ret)
+            return ret;
+        stream->headersDone = 1;
+    }
+
+    if (stream->totalOut < stream->decompSize)
+    {
+        /* We did not decompress everything */
+        ret = yaz0_DoDecompress(stream);
+        if (ret)
+            return ret;
+    }
+
+    /* We did decompress everything */
+    flush(stream);
+    if (stream->window_start != stream->window_end)
+        return YAZ0_NEED_AVAIL_OUT;
+    return YAZ0_OK;
+}

data/libyaz0/src/libyaz0/libyaz0.c

@@ -0,0 +1,64 @@
+#include <stdlib.h>
+#include "libyaz0.h"
+
+int yaz0Init(Yaz0Stream** ptr)
+{
+    Yaz0Stream* s;
+
+    s = malloc(sizeof(*s));
+    if (!s)
+        return YAZ0_OUT_OF_MEMORY;
+    s->mode = MODE_NONE;
+    s->cursorOut = 0;
+    s->decompSize = 0;
+    *ptr = s;
+    return YAZ0_OK;
+}
+
+int yaz0Destroy(Yaz0Stream* stream)
+{
+    free(stream);
+    return YAZ0_OK;
+}
+
+int yaz0Run(Yaz0Stream* s)
+{
+    switch (s->mode)
+    {
+    case MODE_NONE:
+        return YAZ0_OK;
+    case MODE_DECOMPRESS:
+        return yaz0_RunDecompress(s);
+    case MODE_COMPRESS:
+        return yaz0_RunCompress(s);
+    default:
+        unreachable();
+    }
+}
+
+int yaz0Input(Yaz0Stream* stream, const void* data, uint32_t size)
+{
+    stream->in = data;
+    stream->sizeIn = size;
+    stream->cursorIn = 0;
+
+    return YAZ0_OK;
+}
+
+int yaz0Output(Yaz0Stream* stream, void* data, uint32_t size)
+{
+    stream->out = data;
+    stream->sizeOut = size;
+    stream->cursorOut = 0;
+    return YAZ0_OK;
+}
+
+uint32_t yaz0OutputChunkSize(const Yaz0Stream* stream)
+{
+    return stream->cursorOut;
+}
+
+uint32_t yaz0DecompressedSize(const Yaz0Stream* stream)
+{
+    return stream->decompSize;
+}