yaz0 0.3.0 → 0.4.2

data/libyaz0/src/libyaz0/decompress.c

@@ -0,0 +1,197 @@
+#include <string.h>
+#include <stdio.h>
+#include "libyaz0.h"
+
+int yaz0ModeDecompress(Yaz0Stream* s)
+{
+    memset(s, 0, sizeof(*s));
+    s->mode = MODE_DECOMPRESS;
+    return YAZ0_OK;
+}
+
+void loadAux(Yaz0Stream* stream, uint32_t size)
+{
+    if (stream->sizeIn - stream->cursorIn < size)
+        size = stream->sizeIn - stream->cursorIn;
+    for (uint32_t i = 0; i < size; i++)
+        stream->auxBuf[stream->auxSize++] = stream->in[stream->cursorIn++];
+}
+
+int flush(Yaz0Stream* stream)
+{
+    size_t chunkSize;
+    size_t outSize;
+
+    if (stream->window_start == stream->window_end)
+        return YAZ0_OK;
+    if (stream->cursorOut >= stream->sizeOut)
+        return YAZ0_NEED_AVAIL_OUT;
+    outSize = stream->sizeOut - stream->cursorOut;
+    if (stream->window_start > stream->window_end)
+    {
+        /* Wrap around - we will need 2 copies */
+        chunkSize = WINDOW_SIZE - stream->window_start;
+        if (chunkSize > outSize)
+            chunkSize = outSize;
+        memcpy(stream->out + stream->cursorOut, stream->window + stream->window_start, chunkSize);
+        stream->cursorOut += chunkSize;
+        stream->window_start += chunkSize;
+        stream->window_start %= WINDOW_SIZE;
+        outSize -= chunkSize;
+    }
+    /* Copy the rest */
+    chunkSize = stream->window_end - stream->window_start;
+    if (chunkSize > outSize)
+        chunkSize = outSize;
+    memcpy(stream->out + stream->cursorOut, stream->window + stream->window_start, chunkSize);
+    stream->cursorOut += chunkSize;
+    stream->window_start += chunkSize;
+    return YAZ0_OK;
+}
+
+static int yaz0_ReadHeaders(Yaz0Stream* stream)
+{
+    loadAux(stream, 16 - stream->auxSize);
+    if (stream->auxSize < 16)
+        return YAZ0_NEED_AVAIL_IN;
+    if (memcmp(stream->auxBuf, "Yaz0", 4))
+        return YAZ0_BAD_MAGIC;
+    stream->decompSize = swap32(*(uint32_t*)&stream->auxBuf[4]);
+    stream->auxSize = 0;
+    return YAZ0_OK;
+}
+
+static uint32_t windowFreeSize(Yaz0Stream* stream)
+{
+    uint32_t size;
+    if (stream->window_start > stream->window_end)
+        size = WINDOW_SIZE - stream->window_start + stream->window_end;
+    else
+        size = stream->window_end - stream->window_start;
+    return WINDOW_SIZE - size;
+}
+
+static int ensureWindowFree(Yaz0Stream* stream)
+{
+    static const uint32_t maxSize = 0x111 * 8;
+
+    if (windowFreeSize(stream) <= maxSize)
+    {
+        flush(stream);
+        if (windowFreeSize(stream) <= maxSize)
+            return YAZ0_NEED_AVAIL_OUT;
+    }
+    return YAZ0_OK;
+}
+
+int yaz0_DoDecompress(Yaz0Stream* stream)
+{
+    uint8_t     groupBit;
+    uint8_t     byte;
+    int         ret;
+    uint16_t    n;
+    uint16_t    r;
+    for (;;)
+    {
+        /* No group and no EOF - We need to read */
+        if (stream->groupCount == 0)
+        {
+            /* Before we read, we eant to ensure the window is large enough */
+            /* This will avoid a lot of checks and let us write the whole group */
+            ret = ensureWindowFree(stream);
+            if (ret)
+                return ret;
+            if (stream->cursorIn >= stream->sizeIn)
+                return YAZ0_NEED_AVAIL_IN;
+            stream->groupHeader = stream->in[stream->cursorIn++];
+            stream->groupCount = 8;
+        }
+
+        /* We have a group! */
+        while (stream->groupCount)
+        {
+            /* Get the group bit */
+            groupBit = stream->groupHeader & (1 << (stream->groupCount - 1));
+            if (groupBit)
+            {
+                /* Group bit is 1 - direct write */
+                if (stream->cursorIn >= stream->sizeIn)
+                    return YAZ0_NEED_AVAIL_IN;
+
+                /* Everything ok, copy the byte */
+                byte = stream->in[stream->cursorIn++];
+                stream->window[stream->window_end++] = byte;
+                stream->window_end %= WINDOW_SIZE;
+                stream->totalOut++;
+            }
+            else
+            {
+                if (stream->auxSize < 2)
+                {
+                    loadAux(stream, 2 - stream->auxSize);
+                    if (stream->auxSize < 2)
+                        return YAZ0_NEED_AVAIL_IN;
+                }
+                /* Do we need 2 or 3 bytes? */
+                n = ((uint8_t)stream->auxBuf[0]) >> 4;
+                if (!n)
+                {
+                    /* We have a large chunk */
+                    loadAux(stream, 1);
+                    if (stream->auxSize < 3)
+                        return YAZ0_NEED_AVAIL_IN;
+                    n = (uint8_t)stream->auxBuf[2] + 0x12;
+                }
+                else
+                {
+                    /* We have a small chunk */
+                    n = ((uint8_t)stream->auxBuf[0] >> 4) + 2;
+                }
+                r = ((uint16_t)(((uint8_t)stream->auxBuf[0] & 0x0f) << 8) | ((uint8_t)stream->auxBuf[1]));
+                r++;
+                /* Reset the aux buffer */
+                uint32_t cursor = (stream->window_end + WINDOW_SIZE - r) % WINDOW_SIZE;
+                for (int i = 0; i < n; ++i)
+                {
+                    stream->window[stream->window_end++] = stream->window[cursor++];
+                    stream->window_end %= WINDOW_SIZE;
+                    cursor %= WINDOW_SIZE;
+                }
+                stream->auxSize = 0;
+                stream->totalOut += n;
+            }
+            stream->groupCount--;
+            /* Check for EOF */
+            if (stream->totalOut >= stream->decompSize)
+                return YAZ0_OK;
+        }
+    }
+}
+
+int yaz0_RunDecompress(Yaz0Stream* stream)
+{
+    int ret;
+
+    /* Check the headers */
+    if (!stream->headersDone)
+    {
+        ret = yaz0_ReadHeaders(stream);
+        if (ret)
+            return ret;
+        stream->headersDone = 1;
+    }
+
+    if (stream->totalOut < stream->decompSize)
+    {
+        /* We did not decompress everything */
+        ret = yaz0_DoDecompress(stream);
+        if (ret)
+            return ret;
+    }
+
+    /* We did decompress everything */
+    flush(stream);
+    if (stream->window_start != stream->window_end)
+        return YAZ0_NEED_AVAIL_OUT;
+    return YAZ0_OK;
+}

data/libyaz0/src/libyaz0/libyaz0.c

@@ -0,0 +1,64 @@
+#include <stdlib.h>
+#include "libyaz0.h"
+
+int yaz0Init(Yaz0Stream** ptr)
+{
+    Yaz0Stream* s;
+
+    s = malloc(sizeof(*s));
+    if (!s)
+        return YAZ0_OUT_OF_MEMORY;
+    s->mode = MODE_NONE;
+    s->cursorOut = 0;
+    s->decompSize = 0;
+    *ptr = s;
+    return YAZ0_OK;
+}
+
+int yaz0Destroy(Yaz0Stream* stream)
+{
+    free(stream);
+    return YAZ0_OK;
+}
+
+int yaz0Run(Yaz0Stream* s)
+{
+    switch (s->mode)
+    {
+    case MODE_NONE:
+        return YAZ0_OK;
+    case MODE_DECOMPRESS:
+        return yaz0_RunDecompress(s);
+    case MODE_COMPRESS:
+        return yaz0_RunCompress(s);
+    default:
+        unreachable();
+    }
+}
+
+int yaz0Input(Yaz0Stream* stream, const void* data, uint32_t size)
+{
+    stream->in = data;
+    stream->sizeIn = size;
+    stream->cursorIn = 0;
+
+    return YAZ0_OK;
+}
+
+int yaz0Output(Yaz0Stream* stream, void* data, uint32_t size)
+{
+    stream->out = data;
+    stream->sizeOut = size;
+    stream->cursorOut = 0;
+    return YAZ0_OK;
+}
+
+uint32_t yaz0OutputChunkSize(const Yaz0Stream* stream)
+{
+    return stream->cursorOut;
+}
+
+uint32_t yaz0DecompressedSize(const Yaz0Stream* stream)
+{
+    return stream->decompSize;
+}

data/libyaz0/src/libyaz0/libyaz0.h

@@ -0,0 +1,49 @@
+#ifndef LIBYAZ0_H
+#define LIBYAZ0_H
+
+#include <stdint.h>
+#include <yaz0.h>
+
+#define MODE_NONE               0
+#define MODE_DECOMPRESS         1
+#define MODE_COMPRESS           2
+
+#define WINDOW_SIZE             0x4000
+#define HASH_MAX_ENTRIES        0x8000
+#define HASH_REBUILD            0x3000
+
+#define likely(x)       (__builtin_expect((x),1))
+#define unlikely(x)     (__builtin_expect((x),0))
+#define unreachable()   __builtin_unreachable()
+
+struct Yaz0Stream
+{
+    int             mode;
+    int             headersDone;
+    int             level;
+    uint32_t        decompSize;
+    uint32_t        totalOut;
+    const uint8_t*  in;
+    uint8_t*        out;
+    uint32_t        sizeIn;
+    uint32_t        sizeOut;
+    uint32_t        cursorIn;
+    uint32_t        cursorOut;
+    uint8_t         auxBuf[16];
+    uint8_t         auxSize;
+    uint8_t         groupHeader;
+    uint8_t         groupCount;
+    uint32_t        window_start;
+    uint32_t        window_end;
+    uint8_t         window[WINDOW_SIZE];
+    uint32_t        htSize;
+    uint32_t        htHashes[HASH_MAX_ENTRIES];
+    uint32_t        htEntries[HASH_MAX_ENTRIES];
+};
+
+int yaz0_RunDecompress(Yaz0Stream* stream);
+int yaz0_RunCompress(Yaz0Stream* stream);
+
+uint32_t swap32(uint32_t v);
+
+#endif /* LIBYAZ0_H */

data/libyaz0/src/libyaz0/util.c

@@ -0,0 +1,6 @@
+#include "libyaz0.h"
+
+uint32_t swap32(uint32_t in)
+{
+    return ((in & 0xFF) << 24) | ((in & 0xFF00) << 8) | ((in & 0xFF0000) >> 8) | ((in & 0xFF000000) >> 24);
+}

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+require "bundler/setup"
+require "yaz0"
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

data/spec/yaz0_spec.rb

@@ -0,0 +1,39 @@
+RSpec.describe Yaz0 do
+  it "has a version number" do
+    expect(Yaz0::VERSION).not_to be nil
+  end
+
+  it "can compress" do
+    expect(Yaz0.compress("Hello, world")).to be_a(String)
+  end
+
+  it "can decompress" do
+    expect(Yaz0.decompress(Yaz0.compress("Hello, world!"))).to eq("Hello, world!")
+  end
+
+  it "works on very redundent string" do
+    a = "A" * 8192
+    a_compressed = Yaz0.compress(a)
+    expect(a_compressed.size).to be < a.size
+    expect(Yaz0.decompress(a_compressed)).to eq(a)
+  end
+
+  it "compresses identically" do
+    data0 = Yaz0.compress("A" * 8192)
+    data1 = Yaz0.compress("A" * 8192)
+    expect(data0).to eq(data1)
+  end
+
+  it "decompresses identically" do
+    cdata = Yaz0.compress("A" * 8192)
+    data0 = Yaz0.decompress(cdata)
+    data1 = Yaz0.decompress(cdata)
+    expect(data0).to eq(data1)
+  
+    cdata0 = Yaz0.compress("A" * 8192)
+    cdata1 = Yaz0.compress("A" * 8192)
+    data0 = Yaz0.decompress(cdata0)
+    data1 = Yaz0.decompress(cdata1)
+    expect(data0).to eq(data1)
+  end
+end

data/yaz0.gemspec

@@ -14,9 +14,19 @@ Gem::Specification.new do |spec|
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  end
+  spec.files         = Dir[
+    "ext/**/*",
+    "lib/**/*",
+    "spec/**/*",
+    "libyaz0/include/**/*",
+    "libyaz0/src/libyaz0/**/*",
+    "LICENSE.txt",
+    "README.md",
+    "Gemfile",
+    "Rakefile",
+    "yaz0.gemspec",
+    ".rspec"
+  ]
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: yaz0
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.2
 platform: ruby
 authors:
 - Nax
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-05-19 00:00:00.000000000 Z
+date: 2022-08-11 00:00:00.000000000 Z
 dependencies: []
-description: 
+description:
 email:
 - max@bacoux.com
 executables: []
@@ -18,30 +18,33 @@ extensions:
 - ext/yaz0/extconf.rb
 extra_rdoc_files: []
 files:
-- ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/console
-- bin/setup
-- ext/yaz0/buffer.c
-- ext/yaz0/compress.c
-- ext/yaz0/decompress.c
+- ext/yaz0/ext_yaz0.c
+- ext/yaz0/ext_yaz0.h
 - ext/yaz0/extconf.rb
-- ext/yaz0/yaz0.c
-- ext/yaz0/yaz0.h
 - lib/yaz0.rb
 - lib/yaz0/version.rb
+- lib/yaz0/yaz0.so
+- libyaz0/include/yaz0.h
+- libyaz0/src/libyaz0/CMakeLists.txt
+- libyaz0/src/libyaz0/compress.c
+- libyaz0/src/libyaz0/decompress.c
+- libyaz0/src/libyaz0/libyaz0.c
+- libyaz0/src/libyaz0/libyaz0.h
+- libyaz0/src/libyaz0/util.c
+- spec/spec_helper.rb
+- spec/yaz0_spec.rb
 - yaz0.gemspec
 homepage: https://github.com/Nax/ruby-yaz0
 licenses:
 - MIT
 metadata:
   source_code_uri: https://github.com/Nax/ruby-yaz0
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -56,8 +59,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.3.19
+signing_key:
 specification_version: 4
 summary: Compress and decompress Yaz0 data.
 test_files: []

data/.gitignore

@@ -1,19 +0,0 @@
-/.bundle/
-/.yardoc
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-*.bundle
-*.so
-*.o
-*.a
-mkmf.log
-
-# rspec failure tracking
-.rspec_status
-
-/vendor
-/Gemfile.lock

data/.travis.yml

@@ -1,6 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.6.3
-before_install: gem install bundler -v 2.1.4

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "yaz0"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start(__FILE__)

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here

data/ext/yaz0/buffer.c

@@ -1,29 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include "yaz0.h"
-
-void yaz0BufferAlloc(Yaz0Buffer *buf, size_t cap)
-{
-    buf->size = 0;
-    buf->capacity = cap;
-    buf->data = malloc(cap);
-}
-
-void yaz0BufferFree(Yaz0Buffer *buf)
-{
-    free(buf->data);
-    buf->data = NULL;
-}
-
-void yaz0BufferWrite(Yaz0Buffer *buf, const void *data, size_t len)
-{
-    while (buf->size + len > buf->capacity)
-    {
-        buf->capacity = buf->capacity + buf->capacity / 2;
-        buf->data = realloc(buf->data, buf->capacity);
-    }
-
-    memcpy(buf->data + buf->size, data, len);
-    buf->size += len;
-}