yaz0 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ebd330069f0f8a3ec7830485d1b91d3be87f32c72374bfdd68ed3af43730ea7b
+  data.tar.gz: bfb1a68354eb81b6649817f8560e1692236ed384cdcd4780c8434b83348e8c9d
+SHA512:
+  metadata.gz: eae386ef7038fa29f9f46473255c86c5b7c97078cbd5515f93d7d01434d38ba06b6d42779b3f230911aa80d660452e1673c3cd7c5d9e1e944b2d0572f2f6f2fb
+  data.tar.gz: 3e31817e2ab96530e91fda4dfa7881d4571b5d946d64f50dd3ab54e872f68243adf3750b07b88d1e1ffbc0b9ed931712c7d31e04c9e73cb12e09452ec17b4592

data/.gitignore

@@ -0,0 +1,18 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+
+# rspec failure tracking
+.rspec_status
+
+/vendor

data/.rspec

@@ -0,0 +1,3 @@
+--format progress
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.3
+before_install: gem install bundler -v 2.1.4

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in yaz0.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "rake-compiler"
+gem "rspec", "~> 3.0"

data/Gemfile.lock

@@ -0,0 +1,37 @@
+PATH
+  remote: .
+  specs:
+    yaz0 (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.3)
+    rake (12.3.3)
+    rake-compiler (1.1.0)
+      rake
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+
+PLATFORMS
+  x64-mingw32
+
+DEPENDENCIES
+  rake (~> 12.0)
+  rake-compiler
+  rspec (~> 3.0)
+  yaz0!
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Maxime Bacoux
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,18 @@
+# Yaz0
+
+A simple gem to compress/decompress data using the Yaz0 compression algorithm.
+
+## Usage
+
+    require 'yaz0'
+
+    Yaz0.decompress(Yaz0.compress("Hello, world!")) == "Hello, world!"
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/Nax/ruby-yaz0.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,14 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rake/extensiontask"
+
+task :build => :compile
+
+Rake::ExtensionTask.new("yaz0") do |ext|
+  ext.lib_dir = "lib/yaz0"
+end
+
+task :default => [:clobber, :compile, :spec]

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "yaz0"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/ext/yaz0/buffer.c

@@ -0,0 +1,29 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include "yaz0.h"
+
+void yaz0BufferAlloc(Yaz0Buffer *buf, size_t cap)
+{
+    buf->size = 0;
+    buf->capacity = cap;
+    buf->data = malloc(cap);
+}
+
+void yaz0BufferFree(Yaz0Buffer *buf)
+{
+    free(buf->data);
+    buf->data = NULL;
+}
+
+void yaz0BufferWrite(Yaz0Buffer *buf, const void *data, size_t len)
+{
+    while (buf->size + len > buf->capacity)
+    {
+        buf->capacity = buf->capacity + buf->capacity / 2;
+        buf->data = realloc(buf->data, buf->capacity);
+    }
+
+    memcpy(buf->data + buf->size, data, len);
+    buf->size += len;
+}

data/ext/yaz0/compress.c

@@ -0,0 +1,319 @@
+/*
+ * Yaz0
+ *
+ * Compress yaz0 files
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "yaz0.h"
+
+#define RUN_NONE 0
+#define RUN_DATA 1
+#define RUN_REF 2
+
+typedef struct
+{
+    uint8_t type;
+    union {
+        uint8_t data;
+        struct
+        {
+            uint16_t offset;
+            uint16_t len;
+        };
+    };
+} Yaz0Run;
+
+static void makeRunNone(Yaz0Run *r)
+{
+    r->type = RUN_NONE;
+}
+
+static void makeRunData(Yaz0Run *r, uint8_t data)
+{
+    r->type = RUN_DATA;
+    r->data = data;
+}
+
+static void makeRunRef(Yaz0Run *r, uint16_t offset, uint16_t len)
+{
+    r->type = RUN_REF;
+    r->offset = offset;
+    r->len = len;
+}
+
+typedef struct
+{
+    Yaz0Run r[8];
+    uint32_t len;
+    uint8_t group;
+} Yaz0Chunk;
+
+static void makeChunkEmpty(Yaz0Chunk *c)
+{
+    for (int i = 0; i < 8; ++i)
+    {
+        makeRunNone(&(c->r[i]));
+    }
+    c->len = 0;
+    c->group = 0;
+}
+
+typedef struct
+{
+    Yaz0Buffer *dst;
+    const char *data;
+    uint32_t dataSize;
+    uint32_t inCursor;
+} Yaz0Compressor;
+
+static int runCost(const Yaz0Run *run)
+{
+    switch (run->type)
+    {
+    case RUN_NONE:
+        return 1000;
+    case RUN_DATA:
+        return 0;
+    case RUN_REF:
+        return ((run->len >= 0x12) ? 3 : 2) - run->len;
+    }
+    return 0;
+}
+
+static int chunkCost(const Yaz0Chunk *ch)
+{
+    int acc;
+
+    acc = 0;
+    for (int i = 0; i < 8; ++i)
+    {
+        acc += runCost(ch->r + i);
+    }
+
+    return acc;
+}
+
+static void bestChunk(Yaz0Chunk *dst, const Yaz0Chunk *a, const Yaz0Chunk *b)
+{
+    const Yaz0Chunk *src = chunkCost(a) <= chunkCost(b) ? a : b;
+    if (src != dst)
+    {
+        memcpy(dst, src, sizeof(*dst));
+    }
+}
+
+static void bestRun(Yaz0Run *dst, const Yaz0Run *a, const Yaz0Run *b)
+{
+    const Yaz0Run *src = (runCost(a) <= runCost(b)) ? a : b;
+    if (src != dst)
+    {
+        memcpy(dst, src, sizeof(*dst));
+    }
+}
+
+static uint32_t runLength(const Yaz0Run *r)
+{
+    switch (r->type)
+    {
+    case RUN_NONE:
+        return 0;
+    case RUN_DATA:
+        return 1;
+    case RUN_REF:
+        return r->len;
+    }
+    return 0;
+}
+
+static int makeMatchPattern(Yaz0Run *run, Yaz0Compressor *compressor, int len)
+{
+    const char *pattern = compressor->data + compressor->inCursor;
+    int cursorBase = (int)compressor->inCursor - 0x1000;
+    int cursorMax = (int)compressor->inCursor - len;
+
+    if (compressor->inCursor + len > compressor->dataSize)
+        return 0;
+
+    if (cursorBase < 0)
+        cursorBase = 0;
+
+    if (cursorMax < 0)
+        return 0;
+
+    for (int i = cursorMax; i >= cursorBase; --i)
+    {
+        if (memcmp(compressor->data + i, pattern, len) == 0)
+        {
+            makeRunRef(run, compressor->inCursor - i, len);
+            return 1;
+        }
+    }
+    return 0;
+}
+
+static void refineMatch(Yaz0Compressor *compressor, Yaz0Run *run)
+{
+    if (run->type != RUN_REF)
+        return;
+
+    if (run->offset != run->len)
+        return;
+
+    int initialLen = run->len;
+
+    for (;;)
+    {
+        if (run->len == 0x111)
+            return;
+        if (compressor->inCursor + run->len == compressor->dataSize)
+            return;
+        if (compressor->data[compressor->inCursor + run->len] != compressor->data[compressor->inCursor + (run->len % initialLen)])
+            return;
+        run->len++;
+    }
+}
+
+static void makeMatch(Yaz0Run *dst, Yaz0Compressor *compressor)
+{
+    Yaz0Run run;
+
+    for (int i = 1; i <= 0x111; ++i)
+    {
+        if (!makeMatchPattern(&run, compressor, i))
+            break;
+        refineMatch(compressor, &run);
+        bestRun(dst, &run, dst);
+    }
+}
+
+static void makeRun(Yaz0Run *dst, Yaz0Compressor *compressor, int dataOnly)
+{
+    Yaz0Run tmp;
+
+    makeRunNone(dst);
+    if (compressor->inCursor == compressor->dataSize)
+        return;
+    /* Get the basic data run */
+    makeRunData(dst, compressor->data[compressor->inCursor]);
+    if (!dataOnly)
+    {
+        makeRunNone(&tmp);
+        makeMatch(&tmp, compressor);
+        bestRun(dst, &tmp, dst);
+    }
+}
+
+static void writeRun(Yaz0Compressor *c, const Yaz0Run *r)
+{
+    uint8_t tmp8;
+    uint16_t tmp16;
+
+    switch (r->type)
+    {
+    case RUN_NONE:
+        break;
+    case RUN_DATA:
+        yaz0BufferWrite(c->dst, &r->data, 1);
+        break;
+    case RUN_REF:
+        tmp16 = (r->offset - 1);
+        if (r->len <= 0x11)
+        {
+            tmp16 |= ((((uint16_t)r->len) - 2) << 12);
+            tmp16 = swap16(tmp16);
+            yaz0BufferWrite(c->dst, &tmp16, 2);
+        }
+        else
+        {
+            tmp16 = swap16(tmp16);
+            yaz0BufferWrite(c->dst, &tmp16, 2);
+            tmp8 = (r->len - 0x12);
+            yaz0BufferWrite(c->dst, &tmp8, 1);
+        }
+        break;
+    }
+}
+
+static void makeChunkCandidate(Yaz0Chunk *dst, Yaz0Compressor *c, uint8_t mask)
+{
+    uint32_t cursor;
+
+    cursor = c->inCursor;
+    makeChunkEmpty(dst);
+    for (int i = 0; i < 8; ++i)
+    {
+        makeRun(dst->r + i, c, mask & (1 << (7 - i)));
+        if (dst->r[i].type == RUN_DATA)
+        {
+            dst->group |= (1 << (7 - i));
+        }
+        c->inCursor += runLength(dst->r + i);
+        dst->len += runLength(dst->r + i);
+    }
+
+    c->inCursor = cursor;
+}
+
+static void makeChunk(Yaz0Compressor *c)
+{
+    Yaz0Chunk ch;
+    Yaz0Chunk ch2;
+
+    makeChunkCandidate(&ch, c, 0x00);
+    makeChunkCandidate(&ch2, c, 0x01);
+    bestChunk(&ch, &ch, &ch2);
+    makeChunkCandidate(&ch2, c, 0x02);
+    bestChunk(&ch, &ch, &ch2);
+    makeChunkCandidate(&ch2, c, 0x04);
+    bestChunk(&ch, &ch, &ch2);
+    makeChunkCandidate(&ch2, c, 0x08);
+    bestChunk(&ch, &ch, &ch2);
+    makeChunkCandidate(&ch2, c, 0x10);
+    bestChunk(&ch, &ch, &ch2);
+    makeChunkCandidate(&ch2, c, 0x20);
+    bestChunk(&ch, &ch, &ch2);
+    makeChunkCandidate(&ch2, c, 0x40);
+    bestChunk(&ch, &ch, &ch2);
+    makeChunkCandidate(&ch2, c, 0x80);
+    bestChunk(&ch, &ch, &ch2);
+
+    yaz0BufferWrite(c->dst, &ch.group, 1);
+    for (int i = 0; i < 8; ++i)
+    {
+        writeRun(c, ch.r + i);
+    }
+    c->inCursor += ch.len;
+}
+
+static void writeHeader(Yaz0Compressor *c)
+{
+    uint32_t tmp;
+
+    yaz0BufferWrite(c->dst, "Yaz0", 4);
+    tmp = swap32(c->dataSize);
+    yaz0BufferWrite(c->dst, &tmp, 4);
+    tmp = 0;
+    yaz0BufferWrite(c->dst, &tmp, 4);
+    yaz0BufferWrite(c->dst, &tmp, 4);
+}
+
+int yaz0Compress(Yaz0Buffer *dst, const char *data, size_t len)
+{
+    Yaz0Compressor compressor;
+
+    compressor.dst = dst;
+    compressor.data = data;
+    compressor.dataSize = len;
+    compressor.inCursor = 0;
+
+    writeHeader(&compressor);
+
+    while (compressor.inCursor < compressor.dataSize)
+        makeChunk(&compressor);
+    return 1;
+}

data/ext/yaz0/decompress.c

@@ -0,0 +1,64 @@
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include "yaz0.h"
+
+int yaz0Decompress(Yaz0Buffer *dst, const char *d, size_t dataLen)
+{
+    const uint8_t *data;
+    uint32_t dstSize;
+    unsigned inCursor;
+    uint8_t group;
+    int len;
+    int rrr;
+
+    data = d;
+    if (dataLen < 0x10)
+        return 0;
+    if (memcmp(data, "Yaz0", 4) != 0)
+        return 0;
+
+    dstSize = *(uint32_t *)(data + 4);
+    dstSize = swap32(dstSize);
+
+    inCursor = 0x10;
+    dst->data = realloc(dst->data, dstSize);
+
+    while (inCursor < dataLen)
+    {
+        group = data[inCursor++];
+        for (int b = 0; b < 8; ++b)
+        {
+            if (inCursor >= dataLen)
+                break;
+
+            if (group & (1 << (7 - b)))
+            {
+                /* Direct data */
+                dst->data[dst->size++] = data[inCursor++];
+            }
+            else
+            {
+                if (data[inCursor] & 0xf0)
+                {
+                    len = (data[inCursor] >> 4) + 2;
+                    rrr = (((data[inCursor] & 0xf) << 8) | (data[inCursor + 1])) + 1;
+                    inCursor += 2;
+                }
+                else
+                {
+                    rrr = (((data[inCursor] & 0xf) << 8) | (data[inCursor + 1])) + 1;
+                    len = (data[inCursor + 2]) + 0x12;
+                    inCursor += 3;
+                }
+                for (int i = 0; i < len; ++i)
+                {
+                    dst->data[dst->size] = dst->data[dst->size - rrr];
+                    dst->size++;
+                }
+            }
+        }
+    }
+
+    return 1;
+}

data/ext/yaz0/extconf.rb

@@ -0,0 +1,3 @@
+require "mkmf"
+
+create_makefile("yaz0/yaz0")

data/ext/yaz0/yaz0.c

@@ -0,0 +1,48 @@
+#include "yaz0.h"
+#include <stdio.h>
+
+static VALUE compress(VALUE self, VALUE str)
+{
+    Yaz0Buffer buffer;
+    VALUE ret;
+
+    if (TYPE(str) != T_STRING)
+    {
+        rb_raise(rb_eTypeError, "Expected a string");
+        return Qnil;
+    }
+
+    yaz0BufferAlloc(&buffer, 16);
+    yaz0Compress(&buffer, StringValuePtr(str), RSTRING_LEN(str));
+    ret = rb_str_new(buffer.data, buffer.size);
+    yaz0BufferFree(&buffer);
+
+    return ret;
+}
+
+static VALUE decompress(VALUE self, VALUE str)
+{
+    Yaz0Buffer buffer;
+    VALUE ret;
+
+    if (TYPE(str) != T_STRING)
+    {
+        rb_raise(rb_eTypeError, "Expected a string");
+        return Qnil;
+    }
+
+    yaz0BufferAlloc(&buffer, 16);
+    yaz0Decompress(&buffer, StringValuePtr(str), RSTRING_LEN(str));
+    ret = rb_str_new(buffer.data, buffer.size);
+    yaz0BufferFree(&buffer);
+
+    return ret;
+}
+
+void Init_yaz0(void)
+{
+    VALUE mod;
+    mod = rb_define_module("Yaz0");
+    rb_define_module_function(mod, "compress", &compress, 1);
+    rb_define_module_function(mod, "decompress", &decompress, 1);
+}

data/ext/yaz0/yaz0.h

@@ -0,0 +1,38 @@
+#ifndef YAZ0_H
+#define YAZ0_H 1
+
+#include <stddef.h>
+#include "ruby.h"
+
+/*
+ * Swap
+ */
+inline static uint16_t swap16(uint16_t v)
+{
+    return (v << 8) | (v >> 8);
+}
+
+inline static uint32_t swap32(uint32_t v)
+{
+    return (v << 24) | ((v << 8) & 0x00ff0000) | ((v >> 8) & 0x0000ff00) | (v >> 24);
+}
+
+/*
+ * Buffer
+ */
+
+typedef struct
+{
+    size_t size;
+    size_t capacity;
+    char *data;
+} Yaz0Buffer;
+
+void yaz0BufferAlloc(Yaz0Buffer *buf, size_t cap);
+void yaz0BufferFree(Yaz0Buffer *buf);
+void yaz0BufferWrite(Yaz0Buffer *buf, const void *data, size_t len);
+
+int yaz0Compress(Yaz0Buffer *dst, const char *data, size_t len);
+int yaz0Decompress(Yaz0Buffer *dst, const char *data, size_t len);
+
+#endif /* YAZ0_H */

data/lib/yaz0.rb

@@ -0,0 +1,7 @@
+require "yaz0/version"
+require "yaz0/yaz0"
+
+module Yaz0
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/lib/yaz0/version.rb

@@ -0,0 +1,3 @@
+module Yaz0
+  VERSION = "0.1.0"
+end

data/yaz0.gemspec

@@ -0,0 +1,24 @@
+require_relative 'lib/yaz0/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "yaz0"
+  spec.version       = Yaz0::VERSION
+  spec.authors       = ["Nax"]
+  spec.email         = ["max@bacoux.com"]
+
+  spec.summary       = "Compress and decompress Yaz0 data."
+  spec.homepage      = "https://github.com/Nax/ruby-yaz0"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  spec.metadata["source_code_uri"] = "https://github.com/Nax/ruby-yaz0"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.extensions    = ["ext/yaz0/extconf.rb"]
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: yaz0
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nax
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-05-18 00:00:00.000000000 Z
+dependencies: []
+description: 
+email:
+- max@bacoux.com
+executables: []
+extensions:
+- ext/yaz0/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- ext/yaz0/buffer.c
+- ext/yaz0/compress.c
+- ext/yaz0/decompress.c
+- ext/yaz0/extconf.rb
+- ext/yaz0/yaz0.c
+- ext/yaz0/yaz0.h
+- lib/yaz0.rb
+- lib/yaz0/version.rb
+- yaz0.gemspec
+homepage: https://github.com/Nax/ruby-yaz0
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/Nax/ruby-yaz0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Compress and decompress Yaz0 data.
+test_files: []