yaz0 0.3.0 → 0.4.0

data/libyaz0/src/libyaz0/libyaz0.h

@@ -0,0 +1,49 @@
+#ifndef LIBYAZ0_H
+#define LIBYAZ0_H
+
+#include <stdint.h>
+#include <yaz0.h>
+
+#define MODE_NONE               0
+#define MODE_DECOMPRESS         1
+#define MODE_COMPRESS           2
+
+#define WINDOW_SIZE             0x4000
+#define HASH_MAX_ENTRIES        0x8000
+#define HASH_REBUILD            0x3000
+
+#define likely(x)       (__builtin_expect((x),1))
+#define unlikely(x)     (__builtin_expect((x),0))
+#define unreachable()   __builtin_unreachable()
+
+struct Yaz0Stream
+{
+    int             mode;
+    int             headersDone;
+    int             level;
+    uint32_t        decompSize;
+    uint32_t        totalOut;
+    const uint8_t*  in;
+    uint8_t*        out;
+    uint32_t        sizeIn;
+    uint32_t        sizeOut;
+    uint32_t        cursorIn;
+    uint32_t        cursorOut;
+    uint8_t         auxBuf[16];
+    uint8_t         auxSize;
+    uint8_t         groupHeader;
+    uint8_t         groupCount;
+    uint32_t        window_start;
+    uint32_t        window_end;
+    uint8_t         window[WINDOW_SIZE];
+    uint32_t        htSize;
+    uint32_t        htHashes[HASH_MAX_ENTRIES];
+    uint32_t        htEntries[HASH_MAX_ENTRIES];
+};
+
+int yaz0_RunDecompress(Yaz0Stream* stream);
+int yaz0_RunCompress(Yaz0Stream* stream);
+
+uint32_t swap32(uint32_t v);
+
+#endif /* LIBYAZ0_H */

data/libyaz0/src/libyaz0/util.c

@@ -0,0 +1,6 @@
+#include "libyaz0.h"
+
+uint32_t swap32(uint32_t in)
+{
+    return ((in & 0xFF) << 24) | ((in & 0xFF00) << 8) | ((in & 0xFF0000) >> 8) | ((in & 0xFF000000) >> 24);
+}

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+require "bundler/setup"
+require "yaz0"
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

data/spec/yaz0_spec.rb

@@ -0,0 +1,39 @@
+RSpec.describe Yaz0 do
+  it "has a version number" do
+    expect(Yaz0::VERSION).not_to be nil
+  end
+
+  it "can compress" do
+    expect(Yaz0.compress("Hello, world")).to be_a(String)
+  end
+
+  it "can decompress" do
+    expect(Yaz0.decompress(Yaz0.compress("Hello, world!"))).to eq("Hello, world!")
+  end
+
+  it "works on very redundent string" do
+    a = "A" * 8192
+    a_compressed = Yaz0.compress(a)
+    expect(a_compressed.size).to be < a.size
+    expect(Yaz0.decompress(a_compressed)).to eq(a)
+  end
+
+  it "compresses identically" do
+    data0 = Yaz0.compress("A" * 8192)
+    data1 = Yaz0.compress("A" * 8192)
+    expect(data0).to eq(data1)
+  end
+
+  it "decompresses identically" do
+    cdata = Yaz0.compress("A" * 8192)
+    data0 = Yaz0.decompress(cdata)
+    data1 = Yaz0.decompress(cdata)
+    expect(data0).to eq(data1)
+  
+    cdata0 = Yaz0.compress("A" * 8192)
+    cdata1 = Yaz0.compress("A" * 8192)
+    data0 = Yaz0.decompress(cdata0)
+    data1 = Yaz0.decompress(cdata1)
+    expect(data0).to eq(data1)
+  end
+end

data/yaz0.gemspec

@@ -14,9 +14,19 @@ Gem::Specification.new do |spec|
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  end
+  spec.files         = Dir[
+    "ext/**/*",
+    "lib/**/*",
+    "spec/**/*",
+    "libyaz0/include/**/*",
+    "libyaz0/src/libyaz0/**/*",
+    "LICENSE.txt",
+    "README.md",
+    "Gemfile",
+    "Rakefile",
+    "yaz0.gemspec",
+    ".rspec"
+  ]
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: yaz0
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Nax
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-05-19 00:00:00.000000000 Z
+date: 2022-08-09 00:00:00.000000000 Z
 dependencies: []
-description: 
+description:
 email:
 - max@bacoux.com
 executables: []
@@ -18,30 +18,33 @@ extensions:
 - ext/yaz0/extconf.rb
 extra_rdoc_files: []
 files:
-- ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/console
-- bin/setup
-- ext/yaz0/buffer.c
-- ext/yaz0/compress.c
-- ext/yaz0/decompress.c
+- ext/yaz0/ext_yaz0.c
+- ext/yaz0/ext_yaz0.h
 - ext/yaz0/extconf.rb
-- ext/yaz0/yaz0.c
-- ext/yaz0/yaz0.h
 - lib/yaz0.rb
 - lib/yaz0/version.rb
+- lib/yaz0/yaz0.bundle
+- libyaz0/include/yaz0.h
+- libyaz0/src/libyaz0/CMakeLists.txt
+- libyaz0/src/libyaz0/compress.c
+- libyaz0/src/libyaz0/decompress.c
+- libyaz0/src/libyaz0/libyaz0.c
+- libyaz0/src/libyaz0/libyaz0.h
+- libyaz0/src/libyaz0/util.c
+- spec/spec_helper.rb
+- spec/yaz0_spec.rb
 - yaz0.gemspec
 homepage: https://github.com/Nax/ruby-yaz0
 licenses:
 - MIT
 metadata:
   source_code_uri: https://github.com/Nax/ruby-yaz0
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -56,8 +59,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.3.4
+signing_key:
 specification_version: 4
 summary: Compress and decompress Yaz0 data.
 test_files: []

data/.gitignore

@@ -1,19 +0,0 @@
-/.bundle/
-/.yardoc
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-*.bundle
-*.so
-*.o
-*.a
-mkmf.log
-
-# rspec failure tracking
-.rspec_status
-
-/vendor
-/Gemfile.lock

data/.travis.yml

@@ -1,6 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.6.3
-before_install: gem install bundler -v 2.1.4

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "yaz0"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start(__FILE__)

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here

data/ext/yaz0/buffer.c

@@ -1,29 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include "yaz0.h"
-
-void yaz0BufferAlloc(Yaz0Buffer *buf, size_t cap)
-{
-    buf->size = 0;
-    buf->capacity = cap;
-    buf->data = malloc(cap);
-}
-
-void yaz0BufferFree(Yaz0Buffer *buf)
-{
-    free(buf->data);
-    buf->data = NULL;
-}
-
-void yaz0BufferWrite(Yaz0Buffer *buf, const void *data, size_t len)
-{
-    while (buf->size + len > buf->capacity)
-    {
-        buf->capacity = buf->capacity + buf->capacity / 2;
-        buf->data = realloc(buf->data, buf->capacity);
-    }
-
-    memcpy(buf->data + buf->size, data, len);
-    buf->size += len;
-}

data/ext/yaz0/compress.c

@@ -1,319 +0,0 @@
-/*
- * Yaz0
- *
- * Compress yaz0 files
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <string.h>
-
-#include "yaz0.h"
-
-#define RUN_NONE 0
-#define RUN_DATA 1
-#define RUN_REF 2
-
-typedef struct
-{
-    uint8_t type;
-    union {
-        uint8_t data;
-        struct
-        {
-            uint16_t offset;
-            uint16_t len;
-        };
-    };
-} Yaz0Run;
-
-static void makeRunNone(Yaz0Run *r)
-{
-    r->type = RUN_NONE;
-}
-
-static void makeRunData(Yaz0Run *r, uint8_t data)
-{
-    r->type = RUN_DATA;
-    r->data = data;
-}
-
-static void makeRunRef(Yaz0Run *r, uint16_t offset, uint16_t len)
-{
-    r->type = RUN_REF;
-    r->offset = offset;
-    r->len = len;
-}
-
-typedef struct
-{
-    Yaz0Run r[8];
-    uint32_t len;
-    uint8_t group;
-} Yaz0Chunk;
-
-static void makeChunkEmpty(Yaz0Chunk *c)
-{
-    for (int i = 0; i < 8; ++i)
-    {
-        makeRunNone(&(c->r[i]));
-    }
-    c->len = 0;
-    c->group = 0;
-}
-
-typedef struct
-{
-    Yaz0Buffer *dst;
-    const char *data;
-    uint32_t dataSize;
-    int32_t inCursor;
-} Yaz0Compressor;
-
-static int runCost(const Yaz0Run *run)
-{
-    switch (run->type)
-    {
-    case RUN_NONE:
-        return 1000;
-    case RUN_DATA:
-        return 0;
-    case RUN_REF:
-        return ((run->len >= 0x12) ? 3 : 2) - run->len;
-    }
-    return 0;
-}
-
-static int chunkCost(const Yaz0Chunk *ch)
-{
-    int acc;
-
-    acc = 0;
-    for (int i = 0; i < 8; ++i)
-    {
-        acc += runCost(ch->r + i);
-    }
-
-    return acc;
-}
-
-static void bestChunk(Yaz0Chunk *dst, const Yaz0Chunk *a, const Yaz0Chunk *b)
-{
-    const Yaz0Chunk *src = chunkCost(a) <= chunkCost(b) ? a : b;
-    if (src != dst)
-    {
-        memcpy(dst, src, sizeof(*dst));
-    }
-}
-
-static void bestRun(Yaz0Run *dst, const Yaz0Run *a, const Yaz0Run *b)
-{
-    const Yaz0Run *src = (runCost(a) <= runCost(b)) ? a : b;
-    if (src != dst)
-    {
-        memcpy(dst, src, sizeof(*dst));
-    }
-}
-
-static uint32_t runLength(const Yaz0Run *r)
-{
-    switch (r->type)
-    {
-    case RUN_NONE:
-        return 0;
-    case RUN_DATA:
-        return 1;
-    case RUN_REF:
-        return r->len;
-    }
-    return 0;
-}
-
-static int makeMatchPattern(Yaz0Run *run, Yaz0Compressor *compressor, int len)
-{
-    const char *pattern = compressor->data + compressor->inCursor;
-    int cursorBase = (int)compressor->inCursor - 0x1000;
-    int cursorMax = (int)compressor->inCursor - len;
-
-    if (compressor->inCursor + len > compressor->dataSize)
-        return 0;
-
-    if (cursorBase < 0)
-        cursorBase = 0;
-
-    if (cursorMax < 0)
-        return 0;
-
-    for (int i = cursorMax; i >= cursorBase; --i)
-    {
-        if (memcmp(compressor->data + i, pattern, len) == 0)
-        {
-            makeRunRef(run, compressor->inCursor - i, len);
-            return 1;
-        }
-    }
-    return 0;
-}
-
-static void refineMatch(Yaz0Compressor *compressor, Yaz0Run *run)
-{
-    if (run->type != RUN_REF)
-        return;
-
-    if (run->offset != run->len)
-        return;
-
-    int initialLen = run->len;
-
-    for (;;)
-    {
-        if (run->len == 0x111)
-            return;
-        if (compressor->inCursor + run->len == compressor->dataSize)
-            return;
-        if (compressor->data[compressor->inCursor + run->len] != compressor->data[compressor->inCursor + (run->len % initialLen)])
-            return;
-        run->len++;
-    }
-}
-
-static void makeMatch(Yaz0Run *dst, Yaz0Compressor *compressor)
-{
-    Yaz0Run run;
-
-    for (int i = 1; i <= 0x111; ++i)
-    {
-        if (!makeMatchPattern(&run, compressor, i))
-            break;
-        refineMatch(compressor, &run);
-        bestRun(dst, &run, dst);
-    }
-}
-
-static void makeRun(Yaz0Run *dst, Yaz0Compressor *compressor, int dataOnly)
-{
-    Yaz0Run tmp;
-
-    makeRunNone(dst);
-    if (compressor->inCursor >= compressor->dataSize)
-        return;
-    /* Get the basic data run */
-    makeRunData(dst, compressor->data[compressor->inCursor]);
-    if (!dataOnly)
-    {
-        makeRunNone(&tmp);
-        makeMatch(&tmp, compressor);
-        bestRun(dst, dst, &tmp);
-    }
-}
-
-static void writeRun(Yaz0Compressor *c, const Yaz0Run *r)
-{
-    uint8_t tmp8;
-    uint16_t tmp16;
-
-    switch (r->type)
-    {
-    case RUN_NONE:
-        break;
-    case RUN_DATA:
-        yaz0BufferWrite(c->dst, &r->data, 1);
-        break;
-    case RUN_REF:
-        tmp16 = (r->offset - 1);
-        if (r->len <= 0x11)
-        {
-            tmp16 |= ((((uint16_t)r->len) - 2) << 12);
-            tmp16 = swap16(tmp16);
-            yaz0BufferWrite(c->dst, &tmp16, 2);
-        }
-        else
-        {
-            tmp16 = swap16(tmp16);
-            yaz0BufferWrite(c->dst, &tmp16, 2);
-            tmp8 = (r->len - 0x12);
-            yaz0BufferWrite(c->dst, &tmp8, 1);
-        }
-        break;
-    }
-}
-
-static void makeChunkCandidate(Yaz0Chunk *dst, Yaz0Compressor *c, uint8_t mask)
-{
-    uint32_t cursor;
-
-    cursor = c->inCursor;
-    makeChunkEmpty(dst);
-    for (int i = 0; i < 8; ++i)
-    {
-        makeRun(dst->r + i, c, mask & (1 << (7 - i)));
-        if (dst->r[i].type == RUN_DATA)
-        {
-            dst->group |= (1 << (7 - i));
-        }
-        c->inCursor += runLength(dst->r + i);
-        dst->len += runLength(dst->r + i);
-    }
-
-    c->inCursor = cursor;
-}
-
-static void makeChunk(Yaz0Compressor *c)
-{
-    Yaz0Chunk ch;
-    Yaz0Chunk ch2;
-
-    makeChunkCandidate(&ch, c, 0x00);
-    makeChunkCandidate(&ch2, c, 0x01);
-    bestChunk(&ch, &ch, &ch2);
-    makeChunkCandidate(&ch2, c, 0x02);
-    bestChunk(&ch, &ch, &ch2);
-    makeChunkCandidate(&ch2, c, 0x04);
-    bestChunk(&ch, &ch, &ch2);
-    makeChunkCandidate(&ch2, c, 0x08);
-    bestChunk(&ch, &ch, &ch2);
-    makeChunkCandidate(&ch2, c, 0x10);
-    bestChunk(&ch, &ch, &ch2);
-    makeChunkCandidate(&ch2, c, 0x20);
-    bestChunk(&ch, &ch, &ch2);
-    makeChunkCandidate(&ch2, c, 0x40);
-    bestChunk(&ch, &ch, &ch2);
-    makeChunkCandidate(&ch2, c, 0x80);
-    bestChunk(&ch, &ch, &ch2);
-
-    yaz0BufferWrite(c->dst, &ch.group, 1);
-    for (int i = 0; i < 8; ++i)
-    {
-        writeRun(c, ch.r + i);
-    }
-    c->inCursor += ch.len;
-}
-
-static void writeHeader(Yaz0Compressor *c)
-{
-    uint32_t tmp;
-
-    yaz0BufferWrite(c->dst, "Yaz0", 4);
-    tmp = swap32(c->dataSize);
-    yaz0BufferWrite(c->dst, &tmp, 4);
-    tmp = 0;
-    yaz0BufferWrite(c->dst, &tmp, 4);
-    yaz0BufferWrite(c->dst, &tmp, 4);
-}
-
-int yaz0Compress(Yaz0Buffer *dst, const char *data, size_t len)
-{
-    Yaz0Compressor compressor;
-
-    compressor.dst = dst;
-    compressor.data = data;
-    compressor.dataSize = len;
-    compressor.inCursor = 0;
-
-    writeHeader(&compressor);
-
-    while (compressor.inCursor < compressor.dataSize)
-        makeChunk(&compressor);
-    return 1;
-}

data/ext/yaz0/decompress.c

@@ -1,65 +0,0 @@
-#include <stdint.h>
-#include <stdlib.h>
-#include <string.h>
-#include "yaz0.h"
-
-int yaz0Decompress(Yaz0Buffer *dst, const char *d, size_t dataLen)
-{
-    const uint8_t *data;
-    uint32_t dstSize;
-    unsigned inCursor;
-    uint8_t group;
-    int len;
-    int rrr;
-
-    data = d;
-    if (dataLen < 0x10)
-        return 0;
-    if (memcmp(data, "Yaz0", 4) != 0)
-        return 0;
-
-    dstSize = *(uint32_t *)(data + 4);
-    dstSize = swap32(dstSize);
-
-    inCursor = 0x10;
-    dst->data = realloc(dst->data, dstSize);
-    dst->capacity = dstSize;
-
-    while (inCursor < dataLen && dst->size < dstSize)
-    {
-        group = data[inCursor++];
-        for (int b = 0; b < 8; ++b)
-        {
-            if (inCursor >= dataLen || dst->size >= dstSize)
-                break;
-
-            if (group & (1 << (7 - b)))
-            {
-                /* Direct data */
-                dst->data[dst->size++] = data[inCursor++];
-            }
-            else
-            {
-                if (data[inCursor] & 0xf0)
-                {
-                    len = (data[inCursor] >> 4) + 2;
-                    rrr = (((data[inCursor] & 0xf) << 8) | (data[inCursor + 1])) + 1;
-                    inCursor += 2;
-                }
-                else
-                {
-                    rrr = (((data[inCursor] & 0xf) << 8) | (data[inCursor + 1])) + 1;
-                    len = (data[inCursor + 2]) + 0x12;
-                    inCursor += 3;
-                }
-                for (int i = 0; i < len; ++i)
-                {
-                    dst->data[dst->size] = dst->data[dst->size - rrr];
-                    dst->size++;
-                }
-            }
-        }
-    }
-
-    return 1;
-}